17 Commits

Author SHA1 Message Date
Miguel Serrano 828b0668cd test 2021-11-03 16:05:22 +01:00
Miguel Serrano 8fa7932b71 test 2021-11-03 16:05:07 +01:00
Miguel Serrano 1a654a2c32 wip 2021-10-20 10:09:31 +02:00
Miguel Serrano 742226a5ea wip 2021-10-20 08:19:55 +02:00
Miguel Serrano 68ea4b3da3 wip 2021-10-20 08:13:53 +02:00
Miguel Serrano 32d27fe95e wip 2021-10-20 07:54:06 +02:00
Miguel Serrano 2a7c510241 wip 2021-10-20 07:46:49 +02:00
Miguel Serrano 76e269f784 wip 2021-10-20 07:42:46 +02:00
Miguel Serrano cc71cebf67 wip 2021-10-20 07:42:36 +02:00
Miguel Serrano 22efbc38dd wip 2021-10-20 07:39:30 +02:00
Miguel Serrano 75ab035918 wip 2021-10-20 07:26:31 +02:00
Miguel Serrano 1f224d7265 wip 2021-10-20 07:24:28 +02:00
Miguel Serrano 5b583822c3 wip 2021-10-20 06:58:29 +02:00
Miguel Serrano 9ab0514039 wip 2021-10-20 06:40:14 +02:00
Miguel Serrano 2663c8db07 added codespace 2021-10-19 11:48:17 +02:00
Miguel Serrano 19236edc3c added codespace 2021-10-19 11:46:11 +02:00
Miguel Serrano fe65fb7138 added codespace 2021-10-19 11:29:18 +02:00
7960 changed files with 761021 additions and 848440 deletions
+41
View File
@@ -0,0 +1,41 @@
FROM ubuntu:20.04
# Makes sure LuaTex cache is writable
# -----------------------------------
ENV TEXMFVAR=/var/lib/sharelatex/tmp/texmf-var
# Install dependencies
# --------------------
ARG DEBIAN_FRONTEND=noninteractive
RUN apt-get update \
&& apt-get install -y \
build-essential wget net-tools unzip time imagemagick optipng strace nginx git python zlib1g-dev libpcre3-dev \
qpdf \
aspell aspell-en aspell-af aspell-am aspell-ar aspell-ar-large aspell-bg aspell-bn aspell-br aspell-ca aspell-cs aspell-cy aspell-da aspell-de aspell-el aspell-eo aspell-es aspell-et aspell-eu-es aspell-fa aspell-fo aspell-fr aspell-ga aspell-gl-minimos aspell-gu aspell-he aspell-hi aspell-hr aspell-hsb aspell-hu aspell-hy aspell-id aspell-is aspell-it aspell-kk aspell-kn aspell-ku aspell-lt aspell-lv aspell-ml aspell-mr aspell-nl aspell-nr aspell-ns aspell-pa aspell-pl aspell-pt aspell-pt-br aspell-ro aspell-ru aspell-sk aspell-sl aspell-ss aspell-st aspell-sv aspell-tl aspell-tn aspell-ts aspell-uk aspell-uz aspell-xh aspell-zu \
\
# install Node.JS 12
&& curl -sSL https://deb.nodesource.com/setup_12.x | bash - \
&& apt-get install -y nodejs \
\
&& rm -rf \
# We are adding a custom nginx config in the main Dockerfile.
/etc/nginx/nginx.conf \
/etc/nginx/sites-enabled/default \
/var/lib/apt/lists/*
# Add envsubst
# ------------
ADD server-ce/vendor/envsubst /usr/bin/envsubst
RUN chmod +x /usr/bin/envsubst
# Install TexLive
# ---------------
# Skipped!
# Set up sharelatex user and home directory
# -----------------------------------------
RUN mkdir -p /var/lib/sharelatex && \
mkdir -p /var/log/sharelatex && \
mkdir -p /var/lib/sharelatex/data/template_files
+17
View File
@@ -0,0 +1,17 @@
{
"name": "Overleaf Community Edition Codespace",
"dockerComposeFile": "docker-compose.dev.yml",
"service": "sharelatex",
"workspaceFolder": "/var/www/sharelatex",
"settings": {
"terminal.integrated.shell.linux": "/bin/bash"
},
"extensions": [
"ms-azuretools.vscode-docker",
],
"forwardPorts": [80]
}
+37
View File
@@ -0,0 +1,37 @@
version: '2.2'
services:
sharelatex:
build:
context: ../
dockerfile: .devcontainer/Dockerfile
entrypoint: "echo hola!"
depends_on:
mongo:
condition: service_healthy
redis:
condition: service_started
ports:
- 80:80
links:
- mongo
- redis
environment:
SHARELATEX_APP_NAME: Overleaf CE Codebase Dev Environment
SHARELATEX_MONGO_URL: mongodb://mongo/sharelatex
SHARELATEX_REDIS_HOST: redis
REDIS_HOST: redis
ENABLED_LINKED_FILE_TYPES: 'project_file,project_output_file'
ENABLE_CONVERSIONS: 'true'
EMAIL_CONFIRMATION_DISABLED: 'true'
mongo:
restart: always
image: mongo:4.2
healthcheck:
test: echo 'db.stats().ok' | mongo localhost:27017/test --quiet
interval: 10s
timeout: 10s
retries: 5
redis:
image: redis:5
-25
View File
@@ -1,25 +0,0 @@
root = true
[*]
charset = utf-8
indent_style = space
indent_size = 2
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
[Makefile]
indent_style = tab
[*.go]
indent_style = tab
[*.{pug,coffee}]
indent_style = tab
[*.{pug,patch}]
trim_trailing_whitespace = false
[Jenkinsfile]
insert_final_newline = false
max_line_length = off
-385
View File
@@ -1,385 +0,0 @@
name: Build and Deploy Verso (prod)
# Production deploy. Triggered only by pushes to the `prod` branch — keep `main`
# for day-to-day work and fast-forward `prod` when a build is stable.
#
# Differences from the test deploy (deploy-verso.yml):
# - Runs in the `verso` namespace (test runs in `test`).
# - Mongo / Redis / app data live on PersistentVolumeClaims and are applied
# idempotently: this workflow NEVER deletes them, so data survives deploys.
# - The replica set is initialised only once.
# - Builds/pushes a distinct image tag (verso:stable) so prod and test never
# clobber each other's image.
# - SMTP comes from the `verso-smtp` Secret (create it with kubectl); email is
# optional so the app still boots before the secret exists.
# - Public self-registration stays off (CE default): friends-only, admin
# creates accounts / sends invites.
#
# Out of band (do once): create the PVCs (server-ce/k8s/verso-prod-pvcs.yaml,
# with your storageClass), the `verso-smtp` Secret, and a verso.alocoq.fr
# Ingress (see server-ce/k8s/verso-prod-ingress.example.yaml) + DNS.
on:
push:
branches:
- prod
workflow_dispatch:
env:
SITE_URL: https://verso.alocoq.fr
jobs:
deploy:
runs-on: native
timeout-minutes: 240
steps:
- name: Build and push Verso prod image with BuildKit
run: |
kubectl -n ci delete job verso-buildkit-prod --ignore-not-found=true --wait=true
cat <<'EOF' | kubectl apply -f -
apiVersion: batch/v1
kind: Job
metadata:
name: verso-buildkit-prod
namespace: ci
spec:
backoffLimit: 0
template:
spec:
restartPolicy: Never
initContainers:
- name: prepare
image: alpine/git:latest
command: ["sh", "-c"]
args:
- |
set -eux
REG=registry.git.svc.cluster.local:5000
git clone --depth 1 --branch prod https://git.alocoq.fr/alois/verso.git /workspace/repo
# Build the base image only when Dockerfile-base changes
# (content-hash tag); otherwise reuse the cached base.
BTAG=$(sha256sum /workspace/repo/server-ce/Dockerfile-base | cut -c1-16)
printf '%s' "$BTAG" > /workspace/base_tag
if wget -qO- "http://$REG/v2/verso-base/tags/list" 2>/dev/null | grep -q "\"base-$BTAG\""; then
echo "Base image base-$BTAG already present - skipping base build"
else
touch /workspace/build-base
echo "Base image base-$BTAG not found - base will be built"
fi
volumeMounts:
- name: workspace
mountPath: /workspace
containers:
- name: buildkit
image: moby/buildkit:latest
securityContext:
privileged: true
command: ["sh", "-c"]
args:
- |
set -eux
REG=registry.git.svc.cluster.local:5000
mkdir -p /etc/buildkit
printf '[registry."%s"]\n http = true\n insecure = true\n' "$REG" > /etc/buildkit/buildkitd.toml
BTAG=$(cat /workspace/base_tag)
BASE_REF="$REG/verso-base:base-$BTAG"
if [ -f /workspace/build-base ]; then
buildctl-daemonless.sh build \
--frontend=dockerfile.v0 \
--local context=/workspace/repo \
--local dockerfile=/workspace/repo/server-ce \
--opt filename=Dockerfile-base \
--import-cache type=registry,ref=$REG/verso-cache:base \
--export-cache type=registry,ref=$REG/verso-cache:base,mode=max \
--output type=image,name=$BASE_REF,push=true,registry.insecure=true
else
echo "Reusing existing base image $BASE_REF"
fi
# App image → verso:stable (prod tag).
buildctl-daemonless.sh build \
--frontend=dockerfile.v0 \
--local context=/workspace/repo \
--local dockerfile=/workspace/repo/server-ce \
--opt filename=Dockerfile \
--opt build-arg:OVERLEAF_BASE_TAG=$BASE_REF \
--import-cache type=registry,ref=$REG/verso-cache:app \
--export-cache type=registry,ref=$REG/verso-cache:app,mode=max \
--output type=image,name=$REG/verso:stable,push=true,registry.insecure=true
volumeMounts:
- name: workspace
mountPath: /workspace
volumes:
- name: workspace
emptyDir: {}
EOF
- name: Wait for build
run: |
kubectl -n ci wait --for=condition=complete job/verso-buildkit-prod --timeout=14400s
- name: Show build logs
if: always()
run: |
kubectl -n ci logs job/verso-buildkit-prod -c prepare || true
kubectl -n ci logs job/verso-buildkit-prod -c buildkit || true
- name: Ensure data services (Mongo + Redis, never deleted)
run: |
# Mongo/Redis. Applied idempotently — this step must never delete
# these, so project data survives every deploy. The namespace and the
# PVCs (server-ce/k8s/verso-prod-pvcs.yaml) are provisioned out of
# band, so the runner only needs namespaced rights in `verso` (like
# `test`). This step assumes the namespace and the
# mongo-data / redis-data / verso-data PVCs already exist.
cat <<'EOF' | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
name: mongo
namespace: verso
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app: mongo
template:
metadata:
labels:
app: mongo
spec:
containers:
- name: mongo
image: mongo:8
command: ["mongod", "--replSet", "rs0", "--bind_ip_all"]
ports:
- containerPort: 27017
volumeMounts:
- name: mongo-data
mountPath: /data/db
volumes:
- name: mongo-data
persistentVolumeClaim:
claimName: mongo-data
---
apiVersion: v1
kind: Service
metadata:
name: mongo
namespace: verso
spec:
selector:
app: mongo
ports:
- name: mongo
port: 27017
targetPort: 27017
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: redis
namespace: verso
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app: redis
template:
metadata:
labels:
app: redis
spec:
containers:
- name: redis
image: redis:7
# AOF persistence so a restart doesn't drop in-flight edits
# before they're flushed to Mongo.
command: ["redis-server", "--appendonly", "yes"]
ports:
- containerPort: 6379
volumeMounts:
- name: redis-data
mountPath: /data
volumes:
- name: redis-data
persistentVolumeClaim:
claimName: redis-data
---
apiVersion: v1
kind: Service
metadata:
name: redis
namespace: verso
spec:
selector:
app: redis
ports:
- name: redis
port: 6379
targetPort: 6379
EOF
kubectl -n verso rollout status deployment/mongo --timeout=300s
kubectl -n verso rollout status deployment/redis --timeout=300s
- name: Initialise Mongo replica set (only if not already initialised)
run: |
kubectl -n verso exec deploy/mongo -- mongosh --quiet --eval '
try {
rs.status()
print("replica set already initialised")
} catch (e) {
if (e.codeName === "NotYetInitialized" || /no replset config/i.test(e.message)) {
rs.initiate({ _id: "rs0", members: [{ _id: 0, host: "mongo:27017" }] })
print("replica set initiated")
} else {
throw e
}
}
'
kubectl -n verso exec deploy/mongo -- mongosh --quiet --eval '
while (rs.status().myState !== 1) { sleep(1000) }
print("Mongo replica set is PRIMARY")
'
- name: Ensure Verso deployment + service
run: |
# Stamp the instance name with this build number, e.g. "Verso V0.12 Alpha".
NAV_TITLE="Verso V0.${GITHUB_RUN_NUMBER:-${GITEA_RUN_NUMBER:-0}} Alpha"
cat <<'EOF' | sed "s|__NAV_TITLE__|${NAV_TITLE}|g" | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
name: verso
namespace: verso
spec:
replicas: 1
# RWO data volume → can't run two pods at once; recreate on update.
strategy:
type: Recreate
selector:
matchLabels:
app: verso
template:
metadata:
labels:
app: verso
spec:
securityContext:
# App runs as www-data (uid/gid 33); make the data volume
# group-writable by it.
fsGroup: 33
initContainers:
- name: init-data-perms
image: busybox:latest
command: ["sh", "-c"]
args:
- |
set -eux
mkdir -p /data/template_files /data/user_files \
/data/compiles /data/cache /data/output /data/published
chown -R 33:33 /data
volumeMounts:
- name: verso-data
mountPath: /data
containers:
- name: verso
image: registry.alocoq.fr/verso:stable
# :stable is a fixed tag, so force a pull on every rollout to
# pick up the freshly built image.
imagePullPolicy: Always
ports:
- containerPort: 80
env:
- name: OVERLEAF_MONGO_URL
value: mongodb://mongo:27017/sharelatex?replicaSet=rs0
- name: OVERLEAF_REDIS_HOST
value: redis
- name: REDIS_HOST
value: redis
- name: OVERLEAF_APP_NAME
value: Verso
- name: OVERLEAF_NAV_TITLE
value: "__NAV_TITLE__"
- name: OVERLEAF_SITE_URL
value: https://verso.alocoq.fr
- name: OVERLEAF_SITE_LANGUAGE
value: fr
# Allow anonymous visitors so public published-presentation
# links and read-only share links work without login.
- name: OVERLEAF_ALLOW_PUBLIC_ACCESS
value: "true"
# NB: anonymous read-AND-write sharing is intentionally NOT
# enabled (compiles are unsandboxed → only trusted accounts
# may trigger them). Public self-registration is also off
# (CE default): admin creates accounts / sends invites.
- name: OVERLEAF_ENABLE_PROJECT_PYTHON_VENV
value: "true"
- name: OVERLEAF_LATEX_SHELL_ESCAPE
value: "true"
# (SMTP email vars are loaded below via envFrom.)
# SMTP for password-reset / invite emails. All
# OVERLEAF_EMAIL_* vars come from the optional 'verso-smtp'
# Secret (its keys must be named exactly like those env
# vars). Optional, so the app boots before the secret exists.
envFrom:
- secretRef:
name: verso-smtp
optional: true
volumeMounts:
- name: verso-data
mountPath: /var/lib/overleaf/data
volumes:
- name: verso-data
persistentVolumeClaim:
claimName: verso-data
---
apiVersion: v1
kind: Service
metadata:
name: verso
namespace: verso
spec:
selector:
app: verso
ports:
- name: http
port: 80
targetPort: 80
EOF
- name: Deploy Verso image
run: |
kubectl -n verso set image deployment/verso \
verso=registry.alocoq.fr/verso:stable
kubectl -n verso rollout restart deployment/verso
kubectl -n verso rollout status deployment/verso --timeout=600s
- name: Create initial admin (only if no users exist)
run: |
COUNT=$(kubectl -n verso exec deploy/mongo -- mongosh sharelatex --quiet --eval 'db.users.countDocuments()' | tr -d '[:space:]')
if [ "$COUNT" = "0" ]; then
echo "No users yet — creating the initial admin account"
kubectl -n verso exec deploy/verso -- bash -lc '
cd /overleaf/services/web
node modules/server-ce-scripts/scripts/create-user \
--admin \
--email=alois.coquillard@gmail.com
'
else
echo "Users already exist ($COUNT) — skipping admin creation"
fi
-341
View File
@@ -1,341 +0,0 @@
name: Build and Deploy Verso
on:
push:
branches:
- main
workflow_dispatch:
env:
SITE_URL: https://test.alocoq.fr
jobs:
deploy:
runs-on: native
timeout-minutes: 240
steps:
- name: Build and push Verso images with BuildKit
run: |
kubectl -n ci delete job verso-buildkit --ignore-not-found=true --wait=true
cat <<'EOF' | kubectl apply -f -
apiVersion: batch/v1
kind: Job
metadata:
name: verso-buildkit
namespace: ci
spec:
backoffLimit: 0
template:
spec:
restartPolicy: Never
initContainers:
- name: prepare
image: alpine/git:latest
command: ["sh", "-c"]
args:
- |
set -eux
REG=registry.git.svc.cluster.local:5000
git clone --depth 1 https://git.alocoq.fr/alois/verso.git /workspace/repo
# (#1) Build the base image only when it actually changes.
# The base layers' only repo input is Dockerfile-base, so
# we key on a content hash of that file: the base is tagged
# verso-base:base-<hash> and the app builds FROM that exact
# tag. If a base with this hash is already in the registry,
# the heavy base build (apt, TeX Live, Quarto) is skipped.
BTAG=$(sha256sum /workspace/repo/server-ce/Dockerfile-base | cut -c1-16)
printf '%s' "$BTAG" > /workspace/base_tag
if wget -qO- "http://$REG/v2/verso-base/tags/list" 2>/dev/null | grep -q "\"base-$BTAG\""; then
echo "Base image base-$BTAG already present - skipping base build"
else
touch /workspace/build-base
echo "Base image base-$BTAG not found - base will be built"
fi
volumeMounts:
- name: workspace
mountPath: /workspace
containers:
- name: buildkit
image: moby/buildkit:latest
securityContext:
privileged: true
command: ["sh", "-c"]
args:
- |
set -eux
# Push to the in-cluster registry (plain HTTP) to bypass
# the Traefik ingress, whose read timeout was killing the
# multi-GB TeX Live layer upload mid-stream. Mark the
# registry http+insecure so both push and the base pull
# for the app build treat it as plain HTTP. Written inside
# the container so no extra k8s resources are needed.
REG=registry.git.svc.cluster.local:5000
mkdir -p /etc/buildkit
printf '[registry."%s"]\n http = true\n insecure = true\n' "$REG" > /etc/buildkit/buildkitd.toml
BTAG=$(cat /workspace/base_tag)
BASE_REF="$REG/verso-base:base-$BTAG"
# (#1) Base build, only when prepare flagged it changed.
# (#2) Import/export a registry layer cache so that, when
# the base does change, unchanged layers (e.g. apt) are
# still reused instead of rebuilt from scratch.
if [ -f /workspace/build-base ]; then
buildctl-daemonless.sh build \
--frontend=dockerfile.v0 \
--local context=/workspace/repo \
--local dockerfile=/workspace/repo/server-ce \
--opt filename=Dockerfile-base \
--import-cache type=registry,ref=$REG/verso-cache:base \
--export-cache type=registry,ref=$REG/verso-cache:base,mode=max \
--output type=image,name=$BASE_REF,push=true,registry.insecure=true
else
echo "Reusing existing base image $BASE_REF"
fi
# App image, built FROM the content-pinned base tag.
# (#2) The registry cache lets yarn install be skipped when
# package.json is unchanged; the web build only re-runs
# when the frontend source actually changes.
buildctl-daemonless.sh build \
--frontend=dockerfile.v0 \
--local context=/workspace/repo \
--local dockerfile=/workspace/repo/server-ce \
--opt filename=Dockerfile \
--opt build-arg:OVERLEAF_BASE_TAG=$BASE_REF \
--import-cache type=registry,ref=$REG/verso-cache:app \
--export-cache type=registry,ref=$REG/verso-cache:app,mode=max \
--output type=image,name=$REG/verso:latest,push=true,registry.insecure=true
volumeMounts:
- name: workspace
mountPath: /workspace
volumes:
- name: workspace
emptyDir: {}
EOF
- name: Wait for build
run: |
kubectl -n ci wait --for=condition=complete job/verso-buildkit --timeout=14400s
- name: Show build logs
if: always()
run: |
kubectl -n ci logs job/verso-buildkit -c prepare || true
kubectl -n ci logs job/verso-buildkit -c buildkit || true
- name: Recreate test dependencies
run: |
kubectl -n test delete deployment mongo redis --ignore-not-found=true --wait=true
kubectl -n test delete service mongo redis --ignore-not-found=true --wait=true
cat <<'EOF' | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
name: mongo
namespace: test
spec:
replicas: 1
selector:
matchLabels:
app: mongo
template:
metadata:
labels:
app: mongo
spec:
containers:
- name: mongo
image: mongo:8
command: ["mongod", "--replSet", "rs0", "--bind_ip_all"]
ports:
- containerPort: 27017
volumeMounts:
- name: mongo-data
mountPath: /data/db
volumes:
- name: mongo-data
emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
name: mongo
namespace: test
spec:
selector:
app: mongo
ports:
- name: mongo
port: 27017
targetPort: 27017
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: redis
namespace: test
spec:
replicas: 1
selector:
matchLabels:
app: redis
template:
metadata:
labels:
app: redis
spec:
containers:
- name: redis
image: redis:7
ports:
- containerPort: 6379
volumeMounts:
- name: redis-data
mountPath: /data
volumes:
- name: redis-data
emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
name: redis
namespace: test
spec:
selector:
app: redis
ports:
- name: redis
port: 6379
targetPort: 6379
EOF
kubectl -n test rollout status deployment/mongo --timeout=180s
kubectl -n test rollout status deployment/redis --timeout=180s
sleep 5
kubectl -n test exec deploy/mongo -- mongosh --eval '
rs.initiate({
_id: "rs0",
members: [{ _id: 0, host: "mongo:27017" }]
})
'
kubectl -n test exec deploy/mongo -- mongosh --eval '
while (rs.status().myState !== 1) {
sleep(1000)
}
print("Mongo replica set is PRIMARY")
'
- name: Ensure Verso deployment exists
run: |
# Stamp the instance name with this build number, e.g. "Verso V0.83 Alpha".
NAV_TITLE="Verso V0.${GITHUB_RUN_NUMBER:-${GITEA_RUN_NUMBER:-0}} Alpha"
cat <<'EOF' | sed "s|__NAV_TITLE__|${NAV_TITLE}|g" | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
name: verso
namespace: test
spec:
replicas: 1
selector:
matchLabels:
app: verso
template:
metadata:
labels:
app: verso
spec:
containers:
- name: verso
# Pull via the public address: the cluster nodes' containerd
# is configured for registry.alocoq.fr, not the in-cluster
# service name. Both front the same registry storage, so the
# image pushed via the in-cluster address resolves here too.
image: registry.alocoq.fr/verso:latest
ports:
- containerPort: 80
env:
- name: OVERLEAF_MONGO_URL
value: mongodb://mongo:27017/sharelatex?replicaSet=rs0
- name: OVERLEAF_REDIS_HOST
value: redis
- name: REDIS_HOST
value: redis
- name: OVERLEAF_APP_NAME
value: Verso
- name: OVERLEAF_NAV_TITLE
value: "__NAV_TITLE__"
- name: OVERLEAF_SITE_URL
value: https://test.alocoq.fr
# Default UI language for the instance.
- name: OVERLEAF_SITE_LANGUAGE
value: fr
# Allow anonymous visitors to reach the site so link
# sharing and public presentation links work without a
# login. Per-project and per-route access checks still
# apply; private presentation links still require login.
- name: OVERLEAF_ALLOW_PUBLIC_ACCESS
value: "true"
# Also let anonymous visitors use read-AND-write share
# links (edit without an account). Read-only links only
# need OVERLEAF_ALLOW_PUBLIC_ACCESS above.
- name: OVERLEAF_ALLOW_ANONYMOUS_READ_AND_WRITE_SHARING
value: "true"
# Let Quarto Python cells use a project's requirements.txt:
# the compiler installs it into a cached venv. Gated to the
# project owner + invited collaborators (never anonymous /
# link-sharing users).
- name: OVERLEAF_ENABLE_PROJECT_PYTHON_VENV
value: "true"
- name: OVERLEAF_LATEX_SHELL_ESCAPE
value: "true"
---
apiVersion: v1
kind: Service
metadata:
name: verso
namespace: test
spec:
selector:
app: verso
ports:
- name: http
port: 80
targetPort: 80
EOF
- name: Deploy Verso image
run: |
kubectl -n test set image deployment/verso \
verso=registry.alocoq.fr/verso:latest
kubectl -n test rollout restart deployment/verso
kubectl -n test rollout status deployment/verso --timeout=300s
- name: Create admin user
run: |
sleep 20
kubectl -n test exec deploy/verso -- bash -lc '
cd /overleaf/services/web
node modules/server-ce-scripts/scripts/create-user \
--admin \
--email=test@example.com || true
'
- name: Cleanup
if: always()
run: |
kubectl -n ci delete job verso-buildkit --ignore-not-found=true --wait=true
+44
View File
@@ -0,0 +1,44 @@
<!--
Note: If you are using www.overleaf.com and have a problem,
or if you would like to request a new feature please contact
the support team at support@overleaf.com
This form should only be used to report bugs in the
Community Edition release of Overleaf.
-->
<!-- BUG REPORT TEMPLATE -->
## Steps to Reproduce
<!-- Describe the steps leading up to when / where you found the bug. -->
<!-- Screenshots may be helpful here. -->
1.
2.
3.
## Expected Behaviour
<!-- What should have happened when you completed the steps above? -->
## Observed Behaviour
<!-- What actually happened when you completed the steps above? -->
<!-- Screenshots may be helpful here. -->
## Context
<!-- How has this issue affected you? What were you trying to accomplish? -->
## Technical Info
<!-- Provide any technical details that may be applicable (or N/A if not applicable). -->
* URL:
* Browser Name and version:
* Operating System and version (desktop or mobile):
* Signed in as:
* Project and/or file:
## Analysis
<!--- Optionally, document investigation of / suggest a fix for the bug, e.g. 'comes from this line / commit' -->
-56
View File
@@ -1,56 +0,0 @@
---
name: Bug report
about: Report a bug
title: ''
labels: type:bug
assignees: ''
---
<!--
Note: If you are using www.overleaf.com and have a problem,
or if you would like to request a new feature please contact
the support team at support@overleaf.com
This form should only be used to report bugs in the
Community Edition release of Overleaf.
-->
<!-- BUG REPORT TEMPLATE -->
## Steps to Reproduce
<!-- Describe the steps leading up to when / where you found the bug. -->
<!-- Screenshots may be helpful here. -->
1.
2.
3.
## Expected Behaviour
<!-- What should have happened when you completed the steps above? -->
## Observed Behaviour
<!-- What actually happened when you completed the steps above? -->
<!-- Screenshots may be helpful here. -->
## Context
<!-- How has this issue affected you? What were you trying to accomplish? -->
## Technical Info
<!-- Provide any technical details that may be applicable (or N/A if not applicable). -->
- URL:
- Browser Name and version:
- Operating System and version (desktop or mobile):
- Signed in as:
- Project and/or file:
## Analysis
<!--- Optionally, document investigation of / suggest a fix for the bug, e.g. 'comes from this line / commit' -->
+3 -3
View File
@@ -1,11 +1,11 @@
## Description
<!-- Goal of the pull request -->
## Related issues / Pull Requests
## Related issues / Pull Requests
<!-- Fixes #xyz, Contributes to #xyz, Related to #xyz-->
## Contributor Agreement
- [ ] I confirm I have signed the [Contributor License Agreement](https://github.com/overleaf/overleaf/blob/main/CONTRIBUTING.md#contributor-license-agreement)
- [ ] I confirm I have signed the [Contributor License Agreement](https://github.com/overleaf/overleaf/blob/master/CONTRIBUTING.md#contributor-license-agreement)
@@ -1,16 +0,0 @@
diff --git a/dist/index.mjs b/dist/index.mjs
index b781835b37a1262510bdd66d8d3b399a076e9d68..312c314186d85bb3bc2ab2620e99bca259ef7167 100644
--- a/dist/index.mjs
+++ b/dist/index.mjs
@@ -44,10 +44,9 @@ import { z as z2 } from "zod/v4";
// src/tool/types.ts
import { z } from "zod/v4";
-var LATEST_PROTOCOL_VERSION = "2025-11-25";
+var LATEST_PROTOCOL_VERSION = "2025-06-18";
var SUPPORTED_PROTOCOL_VERSIONS = [
LATEST_PROTOCOL_VERSION,
- "2025-06-18",
"2025-03-26",
"2024-11-05"
];
@@ -1,831 +0,0 @@
diff --git a/dist/index.cjs b/dist/index.cjs
index 39215ae..b44cb76 100644
--- a/dist/index.cjs
+++ b/dist/index.cjs
@@ -187,16 +187,23 @@ Helper function that returns a transaction spec which inserts a
completion's text in the main selection range, and any other
selection range that has the same text in front of it.
*/
-function insertCompletionText(state$1, text, from, to) {
+function insertCompletionText(state$1, text, from, to, extend) {
let { main } = state$1.selection, fromOff = from - main.from, toOff = to - main.from;
return Object.assign(Object.assign({}, state$1.changeByRange(range => {
if (range != main && from != to &&
state$1.sliceDoc(range.from + fromOff, range.from + toOff) != state$1.sliceDoc(from, to))
return { range };
- let lines = state$1.toText(text);
+ let change = {
+ from: range.from + fromOff,
+ to: to == main.from ? range.to : range.from + toOff,
+ insert: text instanceof state.Text ? text : state$1.toText(text),
+ };
+ if (extend) {
+ extend(state$1, change);
+ }
return {
- changes: { from: range.from + fromOff, to: to == main.from ? range.to : range.from + toOff, insert: lines },
- range: state.EditorSelection.cursor(range.from + fromOff + lines.length)
+ changes: change,
+ range: state.EditorSelection.cursor(change.from + change.insert.length)
};
})), { scrollIntoView: true, userEvent: "input.complete" });
}
@@ -389,7 +396,9 @@ const completionConfig = state.Facet.define({
filterStrict: false,
compareCompletions: (a, b) => a.label.localeCompare(b.label),
interactionDelay: 75,
- updateSyncTime: 100
+ updateSyncTime: 100,
+ // overleaf: default to at top which is default CM6 behaviour
+ unfilteredResultsAtEnd: false
}, {
defaultKeymap: (a, b) => a && b,
closeOnBlur: (a, b) => a && b,
@@ -744,6 +753,7 @@ function score(option) {
(option.type ? 1 : 0);
}
function sortOptions(active, state) {
+ var _a;
let options = [];
let sections = null;
let addOption = (option) => {
@@ -763,7 +773,8 @@ function sortOptions(active, state) {
let getMatch = a.result.getMatch;
if (a.result.filter === false) {
for (let option of a.result.options) {
- addOption(new Option(option, a.source, getMatch ? getMatch(option) : [], 1e9 - options.length));
+ let defaultScore = conf.unfilteredResultsAtEnd ? -1e9 : 1e9;
+ addOption(new Option(option, a.source, getMatch ? getMatch(option) : [], defaultScore - options.length));
}
}
else {
@@ -790,15 +801,42 @@ function sortOptions(active, state) {
}
}
let result = [], prev = null;
+ const priorityIndices = new Map();
let compare = conf.compareCompletions;
for (let opt of options.sort((a, b) => (b.score - a.score) || compare(a.completion, b.completion))) {
+ // overleaf: Deduplicate results with dedup options
+ // The goal is to keep only the highest priority option, in the
+ // highest scoring position.
+ const key = (_a = opt.completion.deduplicate) === null || _a === void 0 ? void 0 : _a.key;
+ if (key) {
+ // Handle merging specifically for deduplicated items item
+ const currentOptionIndex = priorityIndices.get(key);
+ if (currentOptionIndex === undefined) {
+ priorityIndices.set(key, result.length);
+ result.push(opt);
+ prev = opt.completion;
+ }
+ else {
+ if (result[currentOptionIndex].completion.deduplicate.priority < opt.completion.deduplicate.priority) {
+ result[currentOptionIndex] = opt;
+ if (currentOptionIndex === result.length - 1) {
+ prev = opt.completion;
+ }
+ }
+ }
+ continue;
+ }
+ // overleaf: end
let cur = opt.completion;
- if (!prev || prev.label != cur.label || prev.detail != cur.detail ||
- (prev.type != null && cur.type != null && prev.type != cur.type) ||
- prev.apply != cur.apply || prev.boost != cur.boost)
+ if (!prev || prev.label != cur.label)
+ result.push(opt);
+ // overleaf: we're already handling deduplication, so skip extra merges
+ else if (prev.deduplicate)
result.push(opt);
else if (score(opt.completion) > score(prev))
result[result.length - 1] = opt;
+ else if (opt.completion.info)
+ result[result.length - 1] = opt;
prev = opt.completion;
}
return result;
@@ -817,8 +855,9 @@ class CompletionDialog {
: new CompletionDialog(this.options, makeAttrs(id, selected), this.tooltip, this.timestamp, selected, this.disabled);
}
static build(active, state, id, prev, conf, didSetActive) {
- if (prev && !didSetActive && active.some(s => s.isPending))
- return prev.setDisabled();
+ // Overleaf: avoid setting the previous completion state to disabled while completion sources are pending
+ // if (prev && !didSetActive && active.some(s => s.isPending))
+ // return prev.setDisabled()
let options = sortOptions(active, state);
if (!options.length)
return prev && active.some(a => a.isPending) ? prev.setDisabled() : null;
@@ -1017,13 +1056,14 @@ const completionState = state.StateField.define({
view.EditorView.contentAttributes.from(f, state => state.attrs)
]
});
+const getCompletionTooltip = (state) => { var _a; return (_a = state.field(completionState, false)) === null || _a === void 0 ? void 0 : _a.tooltip; };
function applyCompletion(view, option) {
const apply = option.completion.apply || option.completion.label;
let result = view.state.field(completionState).active.find(a => a.source == option.source);
if (!(result instanceof ActiveResult))
return false;
if (typeof apply == "string")
- view.dispatch(Object.assign(Object.assign({}, insertCompletionText(view.state, apply, result.from, result.to)), { annotations: pickedCompletion.of(option.completion) }));
+ view.dispatch(Object.assign(Object.assign({}, insertCompletionText(view.state, apply, result.from, result.to, option.completion.extend)), { annotations: pickedCompletion.of(option.completion) }));
else
apply(view, option.completion, result.from, result.to);
return true;
@@ -1559,20 +1599,42 @@ interpreted as indicating a placeholder.
function snippet(template) {
let snippet = Snippet.parse(template);
return (editor, completion, from, to) => {
- let { text, ranges } = snippet.instantiate(editor.state, from);
- let { main } = editor.state.selection;
- let spec = {
- changes: { from, to: to == main.from ? main.to : to, insert: state.Text.of(text) },
- scrollIntoView: true,
- annotations: completion ? [pickedCompletion.of(completion), state.Transaction.userEvent.of("input.complete")] : undefined
- };
+ let { main } = editor.state.selection, fromOff = from - main.from, toOff = to - main.from;
+ let ranges = [];
+ let totalOffset = 0;
+ let spec = Object.assign(Object.assign({}, editor.state.changeByRange(range => {
+ if (range != main && from != to &&
+ editor.state.sliceDoc(range.from + fromOff, range.from + toOff) != editor.state.sliceDoc(from, to))
+ return { range };
+ let { text, ranges: fieldRanges } = snippet.instantiate(editor.state, range.from + fromOff);
+ let change = {
+ from: range.from + fromOff,
+ to: range.from + toOff,
+ insert: state.Text.of(text)
+ };
+ let originalTo = change.to;
+ let offset = change.insert.length + fromOff;
+ if (completion.extend) {
+ completion.extend(editor.state, change);
+ offset += originalTo - change.to;
+ }
+ for (const fieldRange of fieldRanges) {
+ ranges.push(new FieldRange(fieldRange.field, fieldRange.from + totalOffset, fieldRange.to + totalOffset));
+ }
+ totalOffset += offset;
+ return {
+ changes: change,
+ range: state.EditorSelection.cursor(change.from + change.insert.length)
+ };
+ })), { scrollIntoView: true, annotations: completion ? [pickedCompletion.of(completion), state.Transaction.userEvent.of("input.complete")] : undefined, effects: [] });
if (ranges.length)
spec.selection = fieldSelection(ranges, 0);
if (ranges.some(r => r.field > 0)) {
let active = new ActiveSnippet(ranges, 0);
- let effects = spec.effects = [setActive.of(active)];
- if (editor.state.field(snippetState, false) === undefined)
- effects.push(state.StateEffect.appendConfig.of([snippetState, addSnippetKeymap, snippetPointerHandler, baseTheme]));
+ spec.effects.push(setActive.of(active));
+ if (editor.state.field(snippetState, false) === undefined) {
+ spec.effects.push(state.StateEffect.appendConfig.of([snippetState, addSnippetKeymap, snippetPointerHandler, baseTheme]));
+ }
}
editor.dispatch(editor.state.update(spec));
};
@@ -1746,7 +1808,8 @@ const completeAnyWord = context => {
const defaults = {
brackets: ["(", "[", "{", "'", '"'],
before: ")]}:;>",
- stringPrefixes: []
+ stringPrefixes: [],
+ buildInsert: (state, range, open, close) => open + close,
};
const closeBracketEffect = state.StateEffect.define({
map(value, mapping) {
@@ -1854,8 +1917,8 @@ function insertBracket(state$1, bracket) {
for (let tok of tokens) {
let closed = closing(state.codePointAt(tok, 0));
if (bracket == tok)
- return closed == tok ? handleSame(state$1, tok, tokens.indexOf(tok + tok + tok) > -1, conf)
- : handleOpen(state$1, tok, closed, conf.before || defaults.before);
+ return closed == tok ? handleSame(state$1, tok, tokens.indexOf(tok + tok) > -1, tokens.indexOf(tok + tok + tok) > -1, conf)
+ : handleOpen(state$1, tok, closed, conf.before || defaults.before, conf);
if (bracket == closed && closedBracketAt(state$1, state$1.selection.main.from))
return handleClose(state$1, tok, closed);
}
@@ -1877,17 +1940,21 @@ function prevChar(doc, pos) {
let prev = doc.sliceString(pos - 2, pos);
return state.codePointSize(state.codePointAt(prev, 0)) == prev.length ? prev : prev.slice(1);
}
-function handleOpen(state$1, open, close, closeBefore) {
+function handleOpen(state$1, open, close, closeBefore, config) {
+ let buildInsert = config.buildInsert || defaults.buildInsert;
let dont = null, changes = state$1.changeByRange(range => {
+ var _a;
if (!range.empty)
return { changes: [{ insert: open, from: range.from }, { insert: close, from: range.to }],
effects: closeBracketEffect.of(range.to + open.length),
range: state.EditorSelection.range(range.anchor + open.length, range.head + open.length) };
let next = nextChar(state$1.doc, range.head);
- if (!next || /\s/.test(next) || closeBefore.indexOf(next) > -1)
- return { changes: { insert: open + close, from: range.head },
- effects: closeBracketEffect.of(range.head + open.length),
+ if (!next || /\s/.test(next) || closeBefore.indexOf(next) > -1) {
+ const insert = (_a = buildInsert(state$1, range, open, close)) !== null && _a !== void 0 ? _a : open + close;
+ return { changes: { insert, from: range.head },
+ effects: insert === open ? [] : closeBracketEffect.of(range.head + open.length),
range: state.EditorSelection.cursor(range.head + open.length) };
+ }
return { range: dont = range };
});
return dont ? null : state$1.update(changes, {
@@ -1909,18 +1976,36 @@ function handleClose(state$1, _open, close) {
}
// Handles cases where the open and close token are the same, and
// possibly triple quotes (as in `"""abc"""`-style quoting).
-function handleSame(state$1, token, allowTriple, config) {
+function handleSame(state$1, token, allowDouble, allowTriple, config) {
let stringPrefixes = config.stringPrefixes || defaults.stringPrefixes;
+ let buildInsert = config.buildInsert || defaults.buildInsert;
let dont = null, changes = state$1.changeByRange(range => {
+ var _a, _b, _c;
if (!range.empty)
return { changes: [{ insert: token, from: range.from }, { insert: token, from: range.to }],
effects: closeBracketEffect.of(range.to + token.length),
range: state.EditorSelection.range(range.anchor + token.length, range.head + token.length) };
let pos = range.head, next = nextChar(state$1.doc, pos), start;
- if (next == token) {
+ if (allowTriple && state$1.sliceDoc(pos - 2 * token.length, pos) == token + token &&
+ (start = canStartStringAt(state$1, pos - 2 * token.length, stringPrefixes)) > -1 &&
+ nodeStart(state$1, start)) {
+ return { changes: { insert: token + token + token + token, from: pos },
+ effects: closeBracketEffect.of(pos + token.length),
+ range: state.EditorSelection.cursor(pos + token.length) };
+ }
+ else if (allowDouble && state$1.sliceDoc(pos - token.length, pos) == token &&
+ (start = canStartStringAt(state$1, pos - token.length, stringPrefixes)) > -1 &&
+ nodeStart(state$1, start)) {
+ let insert = (_a = buildInsert(state$1, range, token, token)) !== null && _a !== void 0 ? _a : token + token;
+ return { changes: { insert, from: pos },
+ effects: insert === token ? [] : closeBracketEffect.of(pos + token.length),
+ range: state.EditorSelection.cursor(pos + token.length) };
+ }
+ else if (next == token) {
if (nodeStart(state$1, pos)) {
- return { changes: { insert: token + token, from: pos },
- effects: closeBracketEffect.of(pos + token.length),
+ let insert = (_b = buildInsert(state$1, range, token, token)) !== null && _b !== void 0 ? _b : token + token;
+ return { changes: { insert, from: pos },
+ effects: insert === token ? [] : closeBracketEffect.of(pos + token.length),
range: state.EditorSelection.cursor(pos + token.length) };
}
else if (closedBracketAt(state$1, pos)) {
@@ -1930,18 +2015,13 @@ function handleSame(state$1, token, allowTriple, config) {
range: state.EditorSelection.cursor(pos + content.length) };
}
}
- else if (allowTriple && state$1.sliceDoc(pos - 2 * token.length, pos) == token + token &&
- (start = canStartStringAt(state$1, pos - 2 * token.length, stringPrefixes)) > -1 &&
- nodeStart(state$1, start)) {
- return { changes: { insert: token + token + token + token, from: pos },
- effects: closeBracketEffect.of(pos + token.length),
- range: state.EditorSelection.cursor(pos + token.length) };
- }
else if (state$1.charCategorizer(pos)(next) != state.CharCategory.Word) {
- if (canStartStringAt(state$1, pos, stringPrefixes) > -1 && !probablyInString(state$1, pos, token, stringPrefixes))
- return { changes: { insert: token + token, from: pos },
- effects: closeBracketEffect.of(pos + token.length),
+ if (canStartStringAt(state$1, pos, stringPrefixes) > -1 && !probablyInString(state$1, pos, token, stringPrefixes)) {
+ const insert = (_c = buildInsert(state$1, range, token, token)) !== null && _c !== void 0 ? _c : token + token;
+ return { changes: { insert, from: pos },
+ effects: insert === token ? [] : closeBracketEffect.of(pos + token.length),
range: state.EditorSelection.cursor(pos + token.length) };
+ }
}
return { range: dont = range };
});
@@ -2086,6 +2166,7 @@ exports.completionKeymap = completionKeymap;
exports.completionStatus = completionStatus;
exports.currentCompletions = currentCompletions;
exports.deleteBracketPair = deleteBracketPair;
+exports.getCompletionTooltip = getCompletionTooltip;
exports.hasNextSnippetField = hasNextSnippetField;
exports.hasPrevSnippetField = hasPrevSnippetField;
exports.ifIn = ifIn;
@@ -2093,8 +2174,10 @@ exports.ifNotIn = ifNotIn;
exports.insertBracket = insertBracket;
exports.insertCompletionText = insertCompletionText;
exports.moveCompletionSelection = moveCompletionSelection;
+exports.nextChar = nextChar;
exports.nextSnippetField = nextSnippetField;
exports.pickedCompletion = pickedCompletion;
+exports.prevChar = prevChar;
exports.prevSnippetField = prevSnippetField;
exports.selectedCompletion = selectedCompletion;
exports.selectedCompletionIndex = selectedCompletionIndex;
diff --git a/dist/index.d.cts b/dist/index.d.cts
index b57b8f6..fce47ab 100644
--- a/dist/index.d.cts
+++ b/dist/index.d.cts
@@ -1,6 +1,6 @@
import * as _codemirror_state from '@codemirror/state';
-import { EditorState, ChangeDesc, TransactionSpec, Transaction, StateCommand, Facet, Extension, StateEffect } from '@codemirror/state';
-import { EditorView, Rect, KeyBinding, Command } from '@codemirror/view';
+import { EditorState, Text, ChangeDesc, TransactionSpec, StateCommand, Transaction, Facet, SelectionRange, Extension, StateEffect } from '@codemirror/state';
+import { EditorView, Rect, KeyBinding, Tooltip, Command } from '@codemirror/view';
import * as _lezer_common from '@lezer/common';
/**
@@ -73,6 +73,19 @@ interface Completion {
a `{name}` object.
*/
section?: string | CompletionSection;
+ /**
+ Can be used to alter the change created when the completion is applied
+ */
+ extend?: ExtendCompletion;
+ /**
+ If multiple sources return the same result, use this field to specifiy a
+ deduplication key as well as a priority. For each unique key, only the
+ completion with the highest priority will be shown.
+ */
+ deduplicate?: {
+ key: string;
+ priority: number;
+ };
}
/**
The type returned from
@@ -306,12 +319,17 @@ This annotation is added to transactions that are produced by
picking a completion.
*/
declare const pickedCompletion: _codemirror_state.AnnotationType<Completion>;
+type ExtendCompletion = (state: EditorState, change: {
+ from: number;
+ to: number;
+ insert: string | Text;
+}) => void;
/**
Helper function that returns a transaction spec which inserts a
completion's text in the main selection range, and any other
selection range that has the same text in front of it.
*/
-declare function insertCompletionText(state: EditorState, text: string, from: number, to: number): TransactionSpec;
+declare function insertCompletionText(state: EditorState, text: string | Text, from: number, to: number, extend?: ExtendCompletion): TransactionSpec;
interface CompletionConfig {
/**
@@ -441,6 +459,10 @@ interface CompletionConfig {
milliseconds.
*/
updateSyncTime?: number;
+ /**
+ overleaf: Move unfiltered results after the filtered ones
+ */
+ unfilteredResultsAtEnd?: boolean;
}
/**
@@ -514,6 +536,8 @@ applies the snippet.
*/
declare function snippetCompletion(template: string, completion: Completion): Completion;
+declare const getCompletionTooltip: (state: EditorState) => Tooltip | undefined | null;
+
/**
Returns a command that moves the completion selection forward or
backward by the given amount.
@@ -562,6 +586,11 @@ interface CloseBracketConfig {
these prefixes before the opening quote.
*/
stringPrefixes?: string[];
+ /**
+ An optional callback for overriding the content that's inserted
+ based on surrounding characters
+ */
+ buildInsert?: (state: EditorState, range: SelectionRange, open: string, close: string) => string;
}
/**
Extension to enable bracket-closing behavior. When a closeable
@@ -593,6 +622,8 @@ to programmatically insert brackets—the
take care of running this for user input.)
*/
declare function insertBracket(state: EditorState, bracket: string): Transaction | null;
+declare function nextChar(doc: Text, pos: number): string;
+declare function prevChar(doc: Text, pos: number): string;
/**
Returns an extension that enables autocompletion.
@@ -636,4 +667,5 @@ the currently selected completion.
*/
declare function setSelectedCompletion(index: number): StateEffect<unknown>;
-export { type CloseBracketConfig, type Completion, CompletionContext, type CompletionInfo, type CompletionResult, type CompletionSection, type CompletionSource, acceptCompletion, autocompletion, clearSnippet, closeBrackets, closeBracketsKeymap, closeCompletion, completeAnyWord, completeFromList, completionKeymap, completionStatus, currentCompletions, deleteBracketPair, hasNextSnippetField, hasPrevSnippetField, ifIn, ifNotIn, insertBracket, insertCompletionText, moveCompletionSelection, nextSnippetField, pickedCompletion, prevSnippetField, selectedCompletion, selectedCompletionIndex, setSelectedCompletion, snippet, snippetCompletion, snippetKeymap, startCompletion };
+export { CompletionContext, acceptCompletion, autocompletion, clearSnippet, closeBrackets, closeBracketsKeymap, closeCompletion, completeAnyWord, completeFromList, completionKeymap, completionStatus, currentCompletions, deleteBracketPair, getCompletionTooltip, hasNextSnippetField, hasPrevSnippetField, ifIn, ifNotIn, insertBracket, insertCompletionText, moveCompletionSelection, nextChar, nextSnippetField, pickedCompletion, prevChar, prevSnippetField, selectedCompletion, selectedCompletionIndex, setSelectedCompletion, snippet, snippetCompletion, snippetKeymap, startCompletion };
+export type { CloseBracketConfig, Completion, CompletionInfo, CompletionResult, CompletionSection, CompletionSource };
diff --git a/dist/index.d.ts b/dist/index.d.ts
index b57b8f6..fce47ab 100644
--- a/dist/index.d.ts
+++ b/dist/index.d.ts
@@ -1,6 +1,6 @@
import * as _codemirror_state from '@codemirror/state';
-import { EditorState, ChangeDesc, TransactionSpec, Transaction, StateCommand, Facet, Extension, StateEffect } from '@codemirror/state';
-import { EditorView, Rect, KeyBinding, Command } from '@codemirror/view';
+import { EditorState, Text, ChangeDesc, TransactionSpec, StateCommand, Transaction, Facet, SelectionRange, Extension, StateEffect } from '@codemirror/state';
+import { EditorView, Rect, KeyBinding, Tooltip, Command } from '@codemirror/view';
import * as _lezer_common from '@lezer/common';
/**
@@ -73,6 +73,19 @@ interface Completion {
a `{name}` object.
*/
section?: string | CompletionSection;
+ /**
+ Can be used to alter the change created when the completion is applied
+ */
+ extend?: ExtendCompletion;
+ /**
+ If multiple sources return the same result, use this field to specifiy a
+ deduplication key as well as a priority. For each unique key, only the
+ completion with the highest priority will be shown.
+ */
+ deduplicate?: {
+ key: string;
+ priority: number;
+ };
}
/**
The type returned from
@@ -306,12 +319,17 @@ This annotation is added to transactions that are produced by
picking a completion.
*/
declare const pickedCompletion: _codemirror_state.AnnotationType<Completion>;
+type ExtendCompletion = (state: EditorState, change: {
+ from: number;
+ to: number;
+ insert: string | Text;
+}) => void;
/**
Helper function that returns a transaction spec which inserts a
completion's text in the main selection range, and any other
selection range that has the same text in front of it.
*/
-declare function insertCompletionText(state: EditorState, text: string, from: number, to: number): TransactionSpec;
+declare function insertCompletionText(state: EditorState, text: string | Text, from: number, to: number, extend?: ExtendCompletion): TransactionSpec;
interface CompletionConfig {
/**
@@ -441,6 +459,10 @@ interface CompletionConfig {
milliseconds.
*/
updateSyncTime?: number;
+ /**
+ overleaf: Move unfiltered results after the filtered ones
+ */
+ unfilteredResultsAtEnd?: boolean;
}
/**
@@ -514,6 +536,8 @@ applies the snippet.
*/
declare function snippetCompletion(template: string, completion: Completion): Completion;
+declare const getCompletionTooltip: (state: EditorState) => Tooltip | undefined | null;
+
/**
Returns a command that moves the completion selection forward or
backward by the given amount.
@@ -562,6 +586,11 @@ interface CloseBracketConfig {
these prefixes before the opening quote.
*/
stringPrefixes?: string[];
+ /**
+ An optional callback for overriding the content that's inserted
+ based on surrounding characters
+ */
+ buildInsert?: (state: EditorState, range: SelectionRange, open: string, close: string) => string;
}
/**
Extension to enable bracket-closing behavior. When a closeable
@@ -593,6 +622,8 @@ to programmatically insert brackets—the
take care of running this for user input.)
*/
declare function insertBracket(state: EditorState, bracket: string): Transaction | null;
+declare function nextChar(doc: Text, pos: number): string;
+declare function prevChar(doc: Text, pos: number): string;
/**
Returns an extension that enables autocompletion.
@@ -636,4 +667,5 @@ the currently selected completion.
*/
declare function setSelectedCompletion(index: number): StateEffect<unknown>;
-export { type CloseBracketConfig, type Completion, CompletionContext, type CompletionInfo, type CompletionResult, type CompletionSection, type CompletionSource, acceptCompletion, autocompletion, clearSnippet, closeBrackets, closeBracketsKeymap, closeCompletion, completeAnyWord, completeFromList, completionKeymap, completionStatus, currentCompletions, deleteBracketPair, hasNextSnippetField, hasPrevSnippetField, ifIn, ifNotIn, insertBracket, insertCompletionText, moveCompletionSelection, nextSnippetField, pickedCompletion, prevSnippetField, selectedCompletion, selectedCompletionIndex, setSelectedCompletion, snippet, snippetCompletion, snippetKeymap, startCompletion };
+export { CompletionContext, acceptCompletion, autocompletion, clearSnippet, closeBrackets, closeBracketsKeymap, closeCompletion, completeAnyWord, completeFromList, completionKeymap, completionStatus, currentCompletions, deleteBracketPair, getCompletionTooltip, hasNextSnippetField, hasPrevSnippetField, ifIn, ifNotIn, insertBracket, insertCompletionText, moveCompletionSelection, nextChar, nextSnippetField, pickedCompletion, prevChar, prevSnippetField, selectedCompletion, selectedCompletionIndex, setSelectedCompletion, snippet, snippetCompletion, snippetKeymap, startCompletion };
+export type { CloseBracketConfig, Completion, CompletionInfo, CompletionResult, CompletionSection, CompletionSource };
diff --git a/dist/index.js b/dist/index.js
index 4729223..9361a53 100644
--- a/dist/index.js
+++ b/dist/index.js
@@ -1,4 +1,4 @@
-import { Annotation, StateEffect, EditorSelection, codePointAt, codePointSize, fromCodePoint, Facet, combineConfig, StateField, Prec, Text, Transaction, MapMode, RangeValue, RangeSet, CharCategory } from '@codemirror/state';
+import { Annotation, StateEffect, Text, EditorSelection, codePointAt, codePointSize, fromCodePoint, Facet, combineConfig, StateField, Prec, Transaction, MapMode, RangeValue, RangeSet, CharCategory } from '@codemirror/state';
import { Direction, logException, showTooltip, EditorView, ViewPlugin, getTooltip, Decoration, WidgetType, keymap } from '@codemirror/view';
import { syntaxTree, indentUnit } from '@codemirror/language';
@@ -185,16 +185,23 @@ Helper function that returns a transaction spec which inserts a
completion's text in the main selection range, and any other
selection range that has the same text in front of it.
*/
-function insertCompletionText(state, text, from, to) {
+function insertCompletionText(state, text, from, to, extend) {
let { main } = state.selection, fromOff = from - main.from, toOff = to - main.from;
return Object.assign(Object.assign({}, state.changeByRange(range => {
if (range != main && from != to &&
state.sliceDoc(range.from + fromOff, range.from + toOff) != state.sliceDoc(from, to))
return { range };
- let lines = state.toText(text);
+ let change = {
+ from: range.from + fromOff,
+ to: to == main.from ? range.to : range.from + toOff,
+ insert: text instanceof Text ? text : state.toText(text),
+ };
+ if (extend) {
+ extend(state, change);
+ }
return {
- changes: { from: range.from + fromOff, to: to == main.from ? range.to : range.from + toOff, insert: lines },
- range: EditorSelection.cursor(range.from + fromOff + lines.length)
+ changes: change,
+ range: EditorSelection.cursor(change.from + change.insert.length)
};
})), { scrollIntoView: true, userEvent: "input.complete" });
}
@@ -387,7 +394,9 @@ const completionConfig = /*@__PURE__*/Facet.define({
filterStrict: false,
compareCompletions: (a, b) => a.label.localeCompare(b.label),
interactionDelay: 75,
- updateSyncTime: 100
+ updateSyncTime: 100,
+ // overleaf: default to at top which is default CM6 behaviour
+ unfilteredResultsAtEnd: false
}, {
defaultKeymap: (a, b) => a && b,
closeOnBlur: (a, b) => a && b,
@@ -742,6 +751,7 @@ function score(option) {
(option.type ? 1 : 0);
}
function sortOptions(active, state) {
+ var _a;
let options = [];
let sections = null;
let addOption = (option) => {
@@ -761,7 +771,8 @@ function sortOptions(active, state) {
let getMatch = a.result.getMatch;
if (a.result.filter === false) {
for (let option of a.result.options) {
- addOption(new Option(option, a.source, getMatch ? getMatch(option) : [], 1e9 - options.length));
+ let defaultScore = conf.unfilteredResultsAtEnd ? -1e9 : 1e9;
+ addOption(new Option(option, a.source, getMatch ? getMatch(option) : [], defaultScore - options.length));
}
}
else {
@@ -788,15 +799,42 @@ function sortOptions(active, state) {
}
}
let result = [], prev = null;
+ const priorityIndices = new Map();
let compare = conf.compareCompletions;
for (let opt of options.sort((a, b) => (b.score - a.score) || compare(a.completion, b.completion))) {
+ // overleaf: Deduplicate results with dedup options
+ // The goal is to keep only the highest priority option, in the
+ // highest scoring position.
+ const key = (_a = opt.completion.deduplicate) === null || _a === void 0 ? void 0 : _a.key;
+ if (key) {
+ // Handle merging specifically for deduplicated items item
+ const currentOptionIndex = priorityIndices.get(key);
+ if (currentOptionIndex === undefined) {
+ priorityIndices.set(key, result.length);
+ result.push(opt);
+ prev = opt.completion;
+ }
+ else {
+ if (result[currentOptionIndex].completion.deduplicate.priority < opt.completion.deduplicate.priority) {
+ result[currentOptionIndex] = opt;
+ if (currentOptionIndex === result.length - 1) {
+ prev = opt.completion;
+ }
+ }
+ }
+ continue;
+ }
+ // overleaf: end
let cur = opt.completion;
- if (!prev || prev.label != cur.label || prev.detail != cur.detail ||
- (prev.type != null && cur.type != null && prev.type != cur.type) ||
- prev.apply != cur.apply || prev.boost != cur.boost)
+ if (!prev || prev.label != cur.label)
+ result.push(opt);
+ // overleaf: we're already handling deduplication, so skip extra merges
+ else if (prev.deduplicate)
result.push(opt);
else if (score(opt.completion) > score(prev))
result[result.length - 1] = opt;
+ else if (opt.completion.info)
+ result[result.length - 1] = opt;
prev = opt.completion;
}
return result;
@@ -815,8 +853,9 @@ class CompletionDialog {
: new CompletionDialog(this.options, makeAttrs(id, selected), this.tooltip, this.timestamp, selected, this.disabled);
}
static build(active, state, id, prev, conf, didSetActive) {
- if (prev && !didSetActive && active.some(s => s.isPending))
- return prev.setDisabled();
+ // Overleaf: avoid setting the previous completion state to disabled while completion sources are pending
+ // if (prev && !didSetActive && active.some(s => s.isPending))
+ // return prev.setDisabled()
let options = sortOptions(active, state);
if (!options.length)
return prev && active.some(a => a.isPending) ? prev.setDisabled() : null;
@@ -1015,13 +1054,14 @@ const completionState = /*@__PURE__*/StateField.define({
EditorView.contentAttributes.from(f, state => state.attrs)
]
});
+const getCompletionTooltip = (state) => { var _a; return (_a = state.field(completionState, false)) === null || _a === void 0 ? void 0 : _a.tooltip; };
function applyCompletion(view, option) {
const apply = option.completion.apply || option.completion.label;
let result = view.state.field(completionState).active.find(a => a.source == option.source);
if (!(result instanceof ActiveResult))
return false;
if (typeof apply == "string")
- view.dispatch(Object.assign(Object.assign({}, insertCompletionText(view.state, apply, result.from, result.to)), { annotations: pickedCompletion.of(option.completion) }));
+ view.dispatch(Object.assign(Object.assign({}, insertCompletionText(view.state, apply, result.from, result.to, option.completion.extend)), { annotations: pickedCompletion.of(option.completion) }));
else
apply(view, option.completion, result.from, result.to);
return true;
@@ -1557,20 +1597,42 @@ interpreted as indicating a placeholder.
function snippet(template) {
let snippet = Snippet.parse(template);
return (editor, completion, from, to) => {
- let { text, ranges } = snippet.instantiate(editor.state, from);
- let { main } = editor.state.selection;
- let spec = {
- changes: { from, to: to == main.from ? main.to : to, insert: Text.of(text) },
- scrollIntoView: true,
- annotations: completion ? [pickedCompletion.of(completion), Transaction.userEvent.of("input.complete")] : undefined
- };
+ let { main } = editor.state.selection, fromOff = from - main.from, toOff = to - main.from;
+ let ranges = [];
+ let totalOffset = 0;
+ let spec = Object.assign(Object.assign({}, editor.state.changeByRange(range => {
+ if (range != main && from != to &&
+ editor.state.sliceDoc(range.from + fromOff, range.from + toOff) != editor.state.sliceDoc(from, to))
+ return { range };
+ let { text, ranges: fieldRanges } = snippet.instantiate(editor.state, range.from + fromOff);
+ let change = {
+ from: range.from + fromOff,
+ to: range.from + toOff,
+ insert: Text.of(text)
+ };
+ let originalTo = change.to;
+ let offset = change.insert.length + fromOff;
+ if (completion.extend) {
+ completion.extend(editor.state, change);
+ offset += originalTo - change.to;
+ }
+ for (const fieldRange of fieldRanges) {
+ ranges.push(new FieldRange(fieldRange.field, fieldRange.from + totalOffset, fieldRange.to + totalOffset));
+ }
+ totalOffset += offset;
+ return {
+ changes: change,
+ range: EditorSelection.cursor(change.from + change.insert.length)
+ };
+ })), { scrollIntoView: true, annotations: completion ? [pickedCompletion.of(completion), Transaction.userEvent.of("input.complete")] : undefined, effects: [] });
if (ranges.length)
spec.selection = fieldSelection(ranges, 0);
if (ranges.some(r => r.field > 0)) {
let active = new ActiveSnippet(ranges, 0);
- let effects = spec.effects = [setActive.of(active)];
- if (editor.state.field(snippetState, false) === undefined)
- effects.push(StateEffect.appendConfig.of([snippetState, addSnippetKeymap, snippetPointerHandler, baseTheme]));
+ spec.effects.push(setActive.of(active));
+ if (editor.state.field(snippetState, false) === undefined) {
+ spec.effects.push(StateEffect.appendConfig.of([snippetState, addSnippetKeymap, snippetPointerHandler, baseTheme]));
+ }
}
editor.dispatch(editor.state.update(spec));
};
@@ -1744,7 +1806,8 @@ const completeAnyWord = context => {
const defaults = {
brackets: ["(", "[", "{", "'", '"'],
before: ")]}:;>",
- stringPrefixes: []
+ stringPrefixes: [],
+ buildInsert: (state, range, open, close) => open + close,
};
const closeBracketEffect = /*@__PURE__*/StateEffect.define({
map(value, mapping) {
@@ -1852,8 +1915,8 @@ function insertBracket(state, bracket) {
for (let tok of tokens) {
let closed = closing(codePointAt(tok, 0));
if (bracket == tok)
- return closed == tok ? handleSame(state, tok, tokens.indexOf(tok + tok + tok) > -1, conf)
- : handleOpen(state, tok, closed, conf.before || defaults.before);
+ return closed == tok ? handleSame(state, tok, tokens.indexOf(tok + tok) > -1, tokens.indexOf(tok + tok + tok) > -1, conf)
+ : handleOpen(state, tok, closed, conf.before || defaults.before, conf);
if (bracket == closed && closedBracketAt(state, state.selection.main.from))
return handleClose(state, tok, closed);
}
@@ -1875,17 +1938,21 @@ function prevChar(doc, pos) {
let prev = doc.sliceString(pos - 2, pos);
return codePointSize(codePointAt(prev, 0)) == prev.length ? prev : prev.slice(1);
}
-function handleOpen(state, open, close, closeBefore) {
+function handleOpen(state, open, close, closeBefore, config) {
+ let buildInsert = config.buildInsert || defaults.buildInsert;
let dont = null, changes = state.changeByRange(range => {
+ var _a;
if (!range.empty)
return { changes: [{ insert: open, from: range.from }, { insert: close, from: range.to }],
effects: closeBracketEffect.of(range.to + open.length),
range: EditorSelection.range(range.anchor + open.length, range.head + open.length) };
let next = nextChar(state.doc, range.head);
- if (!next || /\s/.test(next) || closeBefore.indexOf(next) > -1)
- return { changes: { insert: open + close, from: range.head },
- effects: closeBracketEffect.of(range.head + open.length),
+ if (!next || /\s/.test(next) || closeBefore.indexOf(next) > -1) {
+ const insert = (_a = buildInsert(state, range, open, close)) !== null && _a !== void 0 ? _a : open + close;
+ return { changes: { insert, from: range.head },
+ effects: insert === open ? [] : closeBracketEffect.of(range.head + open.length),
range: EditorSelection.cursor(range.head + open.length) };
+ }
return { range: dont = range };
});
return dont ? null : state.update(changes, {
@@ -1907,18 +1974,36 @@ function handleClose(state, _open, close) {
}
// Handles cases where the open and close token are the same, and
// possibly triple quotes (as in `"""abc"""`-style quoting).
-function handleSame(state, token, allowTriple, config) {
+function handleSame(state, token, allowDouble, allowTriple, config) {
let stringPrefixes = config.stringPrefixes || defaults.stringPrefixes;
+ let buildInsert = config.buildInsert || defaults.buildInsert;
let dont = null, changes = state.changeByRange(range => {
+ var _a, _b, _c;
if (!range.empty)
return { changes: [{ insert: token, from: range.from }, { insert: token, from: range.to }],
effects: closeBracketEffect.of(range.to + token.length),
range: EditorSelection.range(range.anchor + token.length, range.head + token.length) };
let pos = range.head, next = nextChar(state.doc, pos), start;
- if (next == token) {
+ if (allowTriple && state.sliceDoc(pos - 2 * token.length, pos) == token + token &&
+ (start = canStartStringAt(state, pos - 2 * token.length, stringPrefixes)) > -1 &&
+ nodeStart(state, start)) {
+ return { changes: { insert: token + token + token + token, from: pos },
+ effects: closeBracketEffect.of(pos + token.length),
+ range: EditorSelection.cursor(pos + token.length) };
+ }
+ else if (allowDouble && state.sliceDoc(pos - token.length, pos) == token &&
+ (start = canStartStringAt(state, pos - token.length, stringPrefixes)) > -1 &&
+ nodeStart(state, start)) {
+ let insert = (_a = buildInsert(state, range, token, token)) !== null && _a !== void 0 ? _a : token + token;
+ return { changes: { insert, from: pos },
+ effects: insert === token ? [] : closeBracketEffect.of(pos + token.length),
+ range: EditorSelection.cursor(pos + token.length) };
+ }
+ else if (next == token) {
if (nodeStart(state, pos)) {
- return { changes: { insert: token + token, from: pos },
- effects: closeBracketEffect.of(pos + token.length),
+ let insert = (_b = buildInsert(state, range, token, token)) !== null && _b !== void 0 ? _b : token + token;
+ return { changes: { insert, from: pos },
+ effects: insert === token ? [] : closeBracketEffect.of(pos + token.length),
range: EditorSelection.cursor(pos + token.length) };
}
else if (closedBracketAt(state, pos)) {
@@ -1928,18 +2013,13 @@ function handleSame(state, token, allowTriple, config) {
range: EditorSelection.cursor(pos + content.length) };
}
}
- else if (allowTriple && state.sliceDoc(pos - 2 * token.length, pos) == token + token &&
- (start = canStartStringAt(state, pos - 2 * token.length, stringPrefixes)) > -1 &&
- nodeStart(state, start)) {
- return { changes: { insert: token + token + token + token, from: pos },
- effects: closeBracketEffect.of(pos + token.length),
- range: EditorSelection.cursor(pos + token.length) };
- }
else if (state.charCategorizer(pos)(next) != CharCategory.Word) {
- if (canStartStringAt(state, pos, stringPrefixes) > -1 && !probablyInString(state, pos, token, stringPrefixes))
- return { changes: { insert: token + token, from: pos },
- effects: closeBracketEffect.of(pos + token.length),
+ if (canStartStringAt(state, pos, stringPrefixes) > -1 && !probablyInString(state, pos, token, stringPrefixes)) {
+ const insert = (_c = buildInsert(state, range, token, token)) !== null && _c !== void 0 ? _c : token + token;
+ return { changes: { insert, from: pos },
+ effects: insert === token ? [] : closeBracketEffect.of(pos + token.length),
range: EditorSelection.cursor(pos + token.length) };
+ }
}
return { range: dont = range };
});
@@ -2071,4 +2151,4 @@ function setSelectedCompletion(index) {
return setSelectedEffect.of(index);
}
-export { CompletionContext, acceptCompletion, autocompletion, clearSnippet, closeBrackets, closeBracketsKeymap, closeCompletion, completeAnyWord, completeFromList, completionKeymap, completionStatus, currentCompletions, deleteBracketPair, hasNextSnippetField, hasPrevSnippetField, ifIn, ifNotIn, insertBracket, insertCompletionText, moveCompletionSelection, nextSnippetField, pickedCompletion, prevSnippetField, selectedCompletion, selectedCompletionIndex, setSelectedCompletion, snippet, snippetCompletion, snippetKeymap, startCompletion };
+export { CompletionContext, acceptCompletion, autocompletion, clearSnippet, closeBrackets, closeBracketsKeymap, closeCompletion, completeAnyWord, completeFromList, completionKeymap, completionStatus, currentCompletions, deleteBracketPair, getCompletionTooltip, hasNextSnippetField, hasPrevSnippetField, ifIn, ifNotIn, insertBracket, insertCompletionText, moveCompletionSelection, nextChar, nextSnippetField, pickedCompletion, prevChar, prevSnippetField, selectedCompletion, selectedCompletionIndex, setSelectedCompletion, snippet, snippetCompletion, snippetKeymap, startCompletion };
@@ -1,381 +0,0 @@
diff --git a/dist/index.cjs b/dist/index.cjs
index 46231ae..fb0f9aa 100644
--- a/dist/index.cjs
+++ b/dist/index.cjs
@@ -592,6 +592,7 @@ class SearchQuery {
this.valid = !!this.search && (!this.regexp || validRegExp(this.search));
this.unquoted = this.unquote(this.search);
this.wholeWord = !!config.wholeWord;
+ this.scope = config.scope;
}
/**
@internal
@@ -606,7 +607,7 @@ class SearchQuery {
eq(other) {
return this.search == other.search && this.replace == other.replace &&
this.caseSensitive == other.caseSensitive && this.regexp == other.regexp &&
- this.wholeWord == other.wholeWord;
+ this.wholeWord == other.wholeWord && this.scope == other.scope;
}
/**
@internal
@@ -631,7 +632,12 @@ class QueryType {
}
}
function stringCursor(spec, state, from, to) {
- return new SearchCursor(state.doc, spec.unquoted, from, to, spec.caseSensitive ? undefined : x => x.toLowerCase(), spec.wholeWord ? stringWordTest(state.doc, state.charCategorizer(state.selection.main.head)) : undefined);
+ const test = spec.wholeWord ? stringWordTest(state.doc, state.charCategorizer(state.selection.main.head)) : undefined;
+ const testWithinScope = (from, to, buffer, bufferPos) => {
+ return (!test || test(from, to, buffer, bufferPos))
+ && (!spec.scope || spec.scope.some(range => from >= range.from && from <= range.to && to >= range.from && to <= range.to));
+ };
+ return new SearchCursor(state.doc, spec.unquoted, from, to, spec.caseSensitive ? undefined : x => x.toLowerCase(), testWithinScope);
}
function stringWordTest(doc, categorizer) {
return (from, to, buf, bufPos) => {
@@ -695,9 +701,14 @@ class StringQuery extends QueryType {
}
}
function regexpCursor(spec, state, from, to) {
+ const test = spec.wholeWord ? regexpWordTest(state.charCategorizer(state.selection.main.head)) : undefined;
+ const testWithinScope = (from, to, match) => {
+ return (!test || test(from, to, match))
+ && (!spec.scope || spec.scope.some(range => from >= range.from && from <= range.to && to >= range.from && to <= range.to));
+ };
return new RegExpCursor(state.doc, spec.search, {
ignoreCase: !spec.caseSensitive,
- test: spec.wholeWord ? regexpWordTest(state.charCategorizer(state.selection.main.head)) : undefined
+ test: testWithinScope,
}, from, to);
}
function charBefore(str, index) {
@@ -737,10 +748,18 @@ class RegExpQuery extends QueryType {
this.prevMatchInRange(state, curTo, state.doc.length);
}
getReplacement(result) {
- return this.spec.unquote(this.spec.replace).replace(/\$([$&\d+])/g, (m, i) => i == "$" ? "$"
- : i == "&" ? result.match[0]
- : i != "0" && +i < result.match.length ? result.match[i]
- : m);
+ return this.spec.unquote(this.spec.replace).replace(/\$([$&]|\d+)/g, (m, i) => {
+ if (i == "&")
+ return result.match[0];
+ if (i == "$")
+ return "$";
+ for (let l = i.length; l > 0; l--) {
+ let n = +i.slice(0, l);
+ if (n > 0 && n < result.match.length)
+ return result.match[n] + i.slice(l);
+ }
+ return m;
+ });
}
matchAll(state, limit) {
let cursor = regexpCursor(this.spec, state, 0, state.doc.length), ranges = [];
@@ -1227,7 +1246,9 @@ const searchExtensions = [
exports.RegExpCursor = RegExpCursor;
exports.SearchCursor = SearchCursor;
exports.SearchQuery = SearchQuery;
+exports.StringQuery = StringQuery;
exports.closeSearchPanel = closeSearchPanel;
+exports.createSearchPanel = createSearchPanel;
exports.findNext = findNext;
exports.findPrevious = findPrevious;
exports.getSearchQuery = getSearchQuery;
@@ -1242,4 +1263,6 @@ exports.searchPanelOpen = searchPanelOpen;
exports.selectMatches = selectMatches;
exports.selectNextOccurrence = selectNextOccurrence;
exports.selectSelectionMatches = selectSelectionMatches;
+exports.selectWord = selectWord;
exports.setSearchQuery = setSearchQuery;
+exports.togglePanel = togglePanel;
diff --git a/dist/index.d.cts b/dist/index.d.cts
index 08f5696..663d192 100644
--- a/dist/index.d.cts
+++ b/dist/index.d.cts
@@ -1,6 +1,6 @@
import * as _codemirror_state from '@codemirror/state';
import { Text, Extension, StateCommand, EditorState, SelectionRange, StateEffect } from '@codemirror/state';
-import { Command, KeyBinding, EditorView, Panel } from '@codemirror/view';
+import { Command, EditorView, Panel, KeyBinding } from '@codemirror/view';
/**
A search cursor provides an iterator over text matches in a
@@ -161,6 +161,7 @@ the `"cm-selectionMatch"` class for the highlighting. When
itself will be highlighted with `"cm-selectionMatch-main"`.
*/
declare function highlightSelectionMatches(options?: HighlightOptions): Extension;
+declare const selectWord: StateCommand;
/**
Select next occurrence of the current selection. Expand selection
to the surrounding word when the selection is empty.
@@ -264,6 +265,13 @@ declare class SearchQuery {
*/
readonly wholeWord: boolean;
/**
+ When set, only include search matches within these ranges
+ */
+ readonly scope?: Readonly<{
+ from: number;
+ to: number;
+ }[]>;
+ /**
Create a query object.
*/
constructor(config: {
@@ -293,6 +301,13 @@ declare class SearchQuery {
Enable whole-word matching.
*/
wholeWord?: boolean;
+ /**
+ The ranges to match within
+ */
+ scope?: Readonly<{
+ from: number;
+ to: number;
+ }[]>;
});
/**
Compare this query to another query.
@@ -307,6 +322,34 @@ declare class SearchQuery {
to: number;
}>;
}
+type SearchResult = typeof SearchCursor.prototype.value;
+declare abstract class QueryType<Result extends SearchResult = SearchResult> {
+ readonly spec: SearchQuery;
+ constructor(spec: SearchQuery);
+ abstract nextMatch(state: EditorState, curFrom: number, curTo: number): Result | null;
+ abstract prevMatch(state: EditorState, curFrom: number, curTo: number): Result | null;
+ abstract getReplacement(result: Result): string;
+ abstract matchAll(state: EditorState, limit: number): readonly Result[] | null;
+ abstract highlight(state: EditorState, from: number, to: number, add: (from: number, to: number) => void): void;
+}
+declare class StringQuery extends QueryType<SearchResult> {
+ constructor(spec: SearchQuery);
+ nextMatch(state: EditorState, curFrom: number, curTo: number): {
+ from: number;
+ to: number;
+ } | null;
+ private prevMatchInRange;
+ prevMatch(state: EditorState, curFrom: number, curTo: number): {
+ from: number;
+ to: number;
+ } | null;
+ getReplacement(_result: SearchResult): string;
+ matchAll(state: EditorState, limit: number): {
+ from: number;
+ to: number;
+ }[] | null;
+ highlight(state: EditorState, from: number, to: number, add: (from: number, to: number) => void): void;
+}
/**
A state effect that updates the current search query. Note that
this only has an effect if the search state has been initialized
@@ -315,6 +358,7 @@ by running [`openSearchPanel`](https://codemirror.net/6/docs/ref/#search.openSea
once).
*/
declare const setSearchQuery: _codemirror_state.StateEffectType<SearchQuery>;
+declare const togglePanel: _codemirror_state.StateEffectType<boolean>;
/**
Get the current search query from an editor state.
*/
@@ -353,6 +397,7 @@ Replace all instances of the search query with the given
replacement.
*/
declare const replaceAll: Command;
+declare function createSearchPanel(view: EditorView): Panel;
/**
Make sure the search panel is open and focused.
*/
@@ -372,4 +417,4 @@ Default search-related key bindings.
*/
declare const searchKeymap: readonly KeyBinding[];
-export { RegExpCursor, SearchCursor, SearchQuery, closeSearchPanel, findNext, findPrevious, getSearchQuery, gotoLine, highlightSelectionMatches, openSearchPanel, replaceAll, replaceNext, search, searchKeymap, searchPanelOpen, selectMatches, selectNextOccurrence, selectSelectionMatches, setSearchQuery };
+export { RegExpCursor, SearchCursor, SearchQuery, StringQuery, closeSearchPanel, createSearchPanel, findNext, findPrevious, getSearchQuery, gotoLine, highlightSelectionMatches, openSearchPanel, replaceAll, replaceNext, search, searchKeymap, searchPanelOpen, selectMatches, selectNextOccurrence, selectSelectionMatches, selectWord, setSearchQuery, togglePanel };
diff --git a/dist/index.d.ts b/dist/index.d.ts
index 08f5696..663d192 100644
--- a/dist/index.d.ts
+++ b/dist/index.d.ts
@@ -1,6 +1,6 @@
import * as _codemirror_state from '@codemirror/state';
import { Text, Extension, StateCommand, EditorState, SelectionRange, StateEffect } from '@codemirror/state';
-import { Command, KeyBinding, EditorView, Panel } from '@codemirror/view';
+import { Command, EditorView, Panel, KeyBinding } from '@codemirror/view';
/**
A search cursor provides an iterator over text matches in a
@@ -161,6 +161,7 @@ the `"cm-selectionMatch"` class for the highlighting. When
itself will be highlighted with `"cm-selectionMatch-main"`.
*/
declare function highlightSelectionMatches(options?: HighlightOptions): Extension;
+declare const selectWord: StateCommand;
/**
Select next occurrence of the current selection. Expand selection
to the surrounding word when the selection is empty.
@@ -264,6 +265,13 @@ declare class SearchQuery {
*/
readonly wholeWord: boolean;
/**
+ When set, only include search matches within these ranges
+ */
+ readonly scope?: Readonly<{
+ from: number;
+ to: number;
+ }[]>;
+ /**
Create a query object.
*/
constructor(config: {
@@ -293,6 +301,13 @@ declare class SearchQuery {
Enable whole-word matching.
*/
wholeWord?: boolean;
+ /**
+ The ranges to match within
+ */
+ scope?: Readonly<{
+ from: number;
+ to: number;
+ }[]>;
});
/**
Compare this query to another query.
@@ -307,6 +322,34 @@ declare class SearchQuery {
to: number;
}>;
}
+type SearchResult = typeof SearchCursor.prototype.value;
+declare abstract class QueryType<Result extends SearchResult = SearchResult> {
+ readonly spec: SearchQuery;
+ constructor(spec: SearchQuery);
+ abstract nextMatch(state: EditorState, curFrom: number, curTo: number): Result | null;
+ abstract prevMatch(state: EditorState, curFrom: number, curTo: number): Result | null;
+ abstract getReplacement(result: Result): string;
+ abstract matchAll(state: EditorState, limit: number): readonly Result[] | null;
+ abstract highlight(state: EditorState, from: number, to: number, add: (from: number, to: number) => void): void;
+}
+declare class StringQuery extends QueryType<SearchResult> {
+ constructor(spec: SearchQuery);
+ nextMatch(state: EditorState, curFrom: number, curTo: number): {
+ from: number;
+ to: number;
+ } | null;
+ private prevMatchInRange;
+ prevMatch(state: EditorState, curFrom: number, curTo: number): {
+ from: number;
+ to: number;
+ } | null;
+ getReplacement(_result: SearchResult): string;
+ matchAll(state: EditorState, limit: number): {
+ from: number;
+ to: number;
+ }[] | null;
+ highlight(state: EditorState, from: number, to: number, add: (from: number, to: number) => void): void;
+}
/**
A state effect that updates the current search query. Note that
this only has an effect if the search state has been initialized
@@ -315,6 +358,7 @@ by running [`openSearchPanel`](https://codemirror.net/6/docs/ref/#search.openSea
once).
*/
declare const setSearchQuery: _codemirror_state.StateEffectType<SearchQuery>;
+declare const togglePanel: _codemirror_state.StateEffectType<boolean>;
/**
Get the current search query from an editor state.
*/
@@ -353,6 +397,7 @@ Replace all instances of the search query with the given
replacement.
*/
declare const replaceAll: Command;
+declare function createSearchPanel(view: EditorView): Panel;
/**
Make sure the search panel is open and focused.
*/
@@ -372,4 +417,4 @@ Default search-related key bindings.
*/
declare const searchKeymap: readonly KeyBinding[];
-export { RegExpCursor, SearchCursor, SearchQuery, closeSearchPanel, findNext, findPrevious, getSearchQuery, gotoLine, highlightSelectionMatches, openSearchPanel, replaceAll, replaceNext, search, searchKeymap, searchPanelOpen, selectMatches, selectNextOccurrence, selectSelectionMatches, setSearchQuery };
+export { RegExpCursor, SearchCursor, SearchQuery, StringQuery, closeSearchPanel, createSearchPanel, findNext, findPrevious, getSearchQuery, gotoLine, highlightSelectionMatches, openSearchPanel, replaceAll, replaceNext, search, searchKeymap, searchPanelOpen, selectMatches, selectNextOccurrence, selectSelectionMatches, selectWord, setSearchQuery, togglePanel };
diff --git a/dist/index.js b/dist/index.js
index 22172ef..08a9974 100644
--- a/dist/index.js
+++ b/dist/index.js
@@ -590,6 +590,7 @@ class SearchQuery {
this.valid = !!this.search && (!this.regexp || validRegExp(this.search));
this.unquoted = this.unquote(this.search);
this.wholeWord = !!config.wholeWord;
+ this.scope = config.scope;
}
/**
@internal
@@ -604,7 +605,7 @@ class SearchQuery {
eq(other) {
return this.search == other.search && this.replace == other.replace &&
this.caseSensitive == other.caseSensitive && this.regexp == other.regexp &&
- this.wholeWord == other.wholeWord;
+ this.wholeWord == other.wholeWord && this.scope == other.scope;
}
/**
@internal
@@ -629,7 +630,12 @@ class QueryType {
}
}
function stringCursor(spec, state, from, to) {
- return new SearchCursor(state.doc, spec.unquoted, from, to, spec.caseSensitive ? undefined : x => x.toLowerCase(), spec.wholeWord ? stringWordTest(state.doc, state.charCategorizer(state.selection.main.head)) : undefined);
+ const test = spec.wholeWord ? stringWordTest(state.doc, state.charCategorizer(state.selection.main.head)) : undefined;
+ const testWithinScope = (from, to, buffer, bufferPos) => {
+ return (!test || test(from, to, buffer, bufferPos))
+ && (!spec.scope || spec.scope.some(range => from >= range.from && from <= range.to && to >= range.from && to <= range.to));
+ };
+ return new SearchCursor(state.doc, spec.unquoted, from, to, spec.caseSensitive ? undefined : x => x.toLowerCase(), testWithinScope);
}
function stringWordTest(doc, categorizer) {
return (from, to, buf, bufPos) => {
@@ -693,9 +699,14 @@ class StringQuery extends QueryType {
}
}
function regexpCursor(spec, state, from, to) {
+ const test = spec.wholeWord ? regexpWordTest(state.charCategorizer(state.selection.main.head)) : undefined;
+ const testWithinScope = (from, to, match) => {
+ return (!test || test(from, to, match))
+ && (!spec.scope || spec.scope.some(range => from >= range.from && from <= range.to && to >= range.from && to <= range.to));
+ };
return new RegExpCursor(state.doc, spec.search, {
ignoreCase: !spec.caseSensitive,
- test: spec.wholeWord ? regexpWordTest(state.charCategorizer(state.selection.main.head)) : undefined
+ test: testWithinScope,
}, from, to);
}
function charBefore(str, index) {
@@ -735,10 +746,18 @@ class RegExpQuery extends QueryType {
this.prevMatchInRange(state, curTo, state.doc.length);
}
getReplacement(result) {
- return this.spec.unquote(this.spec.replace).replace(/\$([$&\d+])/g, (m, i) => i == "$" ? "$"
- : i == "&" ? result.match[0]
- : i != "0" && +i < result.match.length ? result.match[i]
- : m);
+ return this.spec.unquote(this.spec.replace).replace(/\$([$&]|\d+)/g, (m, i) => {
+ if (i == "&")
+ return result.match[0];
+ if (i == "$")
+ return "$";
+ for (let l = i.length; l > 0; l--) {
+ let n = +i.slice(0, l);
+ if (n > 0 && n < result.match.length)
+ return result.match[n] + i.slice(l);
+ }
+ return m;
+ });
}
matchAll(state, limit) {
let cursor = regexpCursor(this.spec, state, 0, state.doc.length), ranges = [];
@@ -1222,4 +1241,4 @@ const searchExtensions = [
baseTheme
];
-export { RegExpCursor, SearchCursor, SearchQuery, closeSearchPanel, findNext, findPrevious, getSearchQuery, gotoLine, highlightSelectionMatches, openSearchPanel, replaceAll, replaceNext, search, searchKeymap, searchPanelOpen, selectMatches, selectNextOccurrence, selectSelectionMatches, setSearchQuery };
+export { RegExpCursor, SearchCursor, SearchQuery, StringQuery, closeSearchPanel, createSearchPanel, findNext, findPrevious, getSearchQuery, gotoLine, highlightSelectionMatches, openSearchPanel, replaceAll, replaceNext, search, searchKeymap, searchPanelOpen, selectMatches, selectNextOccurrence, selectSelectionMatches, selectWord, setSearchQuery, togglePanel };
@@ -1,44 +0,0 @@
diff --git a/lib/read.js b/lib/read.js
index fce6283..6131c31 100644
--- a/lib/read.js
+++ b/lib/read.js
@@ -18,7 +18,7 @@ var iconv = require('iconv-lite')
var onFinished = require('on-finished')
var unpipe = require('unpipe')
var zlib = require('zlib')
-
+var Stream = require('stream')
/**
* Module exports.
*/
@@ -166,25 +166,25 @@ function contentstream (req, debug, inflate) {
case 'deflate':
stream = zlib.createInflate()
debug('inflate body')
- req.pipe(stream)
+ // req.pipe(stream)
break
case 'gzip':
stream = zlib.createGunzip()
debug('gunzip body')
- req.pipe(stream)
+ // req.pipe(stream)
break
case 'identity':
stream = req
stream.length = length
- break
+ return req
default:
throw createError(415, 'unsupported content encoding "' + encoding + '"', {
encoding: encoding,
type: 'encoding.unsupported'
})
}
-
- return stream
+ var pass = new Stream.PassThrough(); Stream.pipeline(req, stream, pass, () => {})
+ return pass
}
/**
@@ -1,13 +0,0 @@
diff --git a/lib/MultiReporters.js b/lib/MultiReporters.js
index 98dc4ef..b2a97bf 100644
--- a/lib/MultiReporters.js
+++ b/lib/MultiReporters.js
@@ -160,7 +160,7 @@ MultiReporters.prototype.getCustomOptions = function (options) {
debug('options file (custom)', customOptionsFile);
try {
- if ('.js' === path.extname(customOptionsFile)) {
+ if (['.js', '.cjs'].includes(path.extname(customOptionsFile))) {
customOptions = require(customOptionsFile);
}
else {
-13
View File
@@ -1,13 +0,0 @@
diff --git a/index.js b/index.js
index b2b6bdd..75e6254 100644
--- a/index.js
+++ b/index.js
@@ -46,7 +46,7 @@ function forwarded (req) {
function getSocketAddr (req) {
return req.socket
? req.socket.remoteAddress
- : req.connection.remoteAddress
+ : req.connection && req.connection.remoteAddress
}
/**
@@ -1,13 +0,0 @@
diff --git a/lib/MultiReporters.js b/lib/MultiReporters.js
index d61e019711d5ac7f82c0fb90548bb0eb41ebbb85..e7a9515e05287621301b831191884a84d72ad0a1 100644
--- a/lib/MultiReporters.js
+++ b/lib/MultiReporters.js
@@ -153,7 +153,7 @@ MultiReporters.prototype.getCustomOptions = function (options) {
debug('options file (custom)', customOptionsFile);
try {
- if ('.js' === path.extname(customOptionsFile)) {
+ if (['.js', '.cjs'].includes(path.extname(customOptionsFile))) {
customOptions = require(customOptionsFile);
}
else {
-13
View File
@@ -1,13 +0,0 @@
diff --git a/lib/make-middleware.js b/lib/make-middleware.js
index ee50988..de77364 100644
--- a/lib/make-middleware.js
+++ b/lib/make-middleware.js
@@ -164,7 +164,7 @@ function makeMiddleware (setup) {
if (fieldname == null) return abortWithCode('MISSING_FIELD_NAME')
// don't attach to the files object, if there is no file
- if (!filename) return fileStream.resume()
+ if (!filename) filename = 'undefined'
// Work around bug in Busboy (https://github.com/mscdex/busboy/issues/6)
if (limits && Object.prototype.hasOwnProperty.call(limits, 'fieldNameSize')) {
-76
View File
@@ -1,76 +0,0 @@
diff --git a/lib/index.js b/lib/index.js
index 567ff5d..8eb45f7 100644
--- a/lib/index.js
+++ b/lib/index.js
@@ -545,8 +545,8 @@ function clone(instance) {
// tee instance body
p1 = new PassThrough();
p2 = new PassThrough();
- body.pipe(p1);
- body.pipe(p2);
+ Stream.pipeline(body, p1, () => {});
+ Stream.pipeline(body, p2, () => {});
// set instance body to teed body and return the other teed body
instance[INTERNALS].body = p1;
body = p2;
@@ -648,14 +648,14 @@ function writeToStream(dest, instance) {
// body is null
dest.end();
} else if (isBlob(body)) {
- body.stream().pipe(dest);
+ Stream.pipeline(body.stream(), dest, () => {});
} else if (Buffer.isBuffer(body)) {
// body is buffer
dest.write(body);
dest.end();
} else {
// body is stream
- body.pipe(dest);
+ Stream.pipeline(body, dest, () => {});
}
}
@@ -1638,7 +1638,7 @@ function fetch(url, opts) {
res.once('end', function () {
if (signal) signal.removeEventListener('abort', abortAndFinalize);
});
- let body = res.pipe(new PassThrough$1());
+ let body = Stream.pipeline(res, new PassThrough(), error => { if (error) reject(error); });
const response_options = {
url: request.url,
@@ -1679,7 +1679,7 @@ function fetch(url, opts) {
// for gzip
if (codings == 'gzip' || codings == 'x-gzip') {
- body = body.pipe(zlib.createGunzip(zlibOptions));
+ body = Stream.pipeline(body, zlib.createGunzip(zlibOptions), error => { if (error) reject(error); });
response = new Response(body, response_options);
resolve(response);
return;
@@ -1689,13 +1689,13 @@ function fetch(url, opts) {
if (codings == 'deflate' || codings == 'x-deflate') {
// handle the infamous raw deflate response from old servers
// a hack for old IIS and Apache servers
- const raw = res.pipe(new PassThrough$1());
+ const raw = Stream.pipeline(res, new PassThrough(), error => { if (error) reject(error); });
raw.once('data', function (chunk) {
// see http://stackoverflow.com/questions/37519828
if ((chunk[0] & 0x0F) === 0x08) {
- body = body.pipe(zlib.createInflate());
+ body = Stream.pipeline(body, zlib.createInflate(), error => { if (error) reject(error); });
} else {
- body = body.pipe(zlib.createInflateRaw());
+ body = Stream.pipeline(body, zlib.createInflateRaw(), error => { if (error) reject(error); });
}
response = new Response(body, response_options);
resolve(response);
@@ -1712,7 +1712,7 @@ function fetch(url, opts) {
// for br
if (codings == 'br' && typeof zlib.createBrotliDecompress === 'function') {
- body = body.pipe(zlib.createBrotliDecompress());
+ body = Stream.pipeline(body, zlib.createBrotliDecompress(), error => { if (error) reject(error); });
response = new Response(body, response_options);
resolve(response);
return;
@@ -1,13 +0,0 @@
diff --git a/lib/utils.js b/lib/utils.js
index 486f9e1..4584507 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -24,7 +24,7 @@ exports.originalURL = function(req, options) {
var trustProxy = options.proxy;
var proto = (req.headers['x-forwarded-proto'] || '').toLowerCase()
- , tls = req.connection.encrypted || (trustProxy && 'https' == proto.split(/\s*,\s*/)[0])
+ , tls = (req.connection && req.connection.encrypted) || (trustProxy && 'https' == proto.split(/\s*,\s*/)[0])
, host = (trustProxy && req.headers['x-forwarded-host']) || req.headers.host
, protocol = tls ? 'https' : 'http'
, path = req.url || '';
-22
View File
@@ -1,22 +0,0 @@
diff --git a/build/pdf.worker.mjs b/build/pdf.worker.mjs
index 6c5c6f1..bb6b7d1 100644
--- a/build/pdf.worker.mjs
+++ b/build/pdf.worker.mjs
@@ -1830,7 +1830,7 @@ async function __wbg_init(module_or_path) {
}
}
if (typeof module_or_path === 'undefined') {
- module_or_path = new URL('qcms_bg.wasm', import.meta.url);
+ module_or_path = new URL(/* webpackIgnore: true */ 'qcms_bg.wasm', import.meta.url);
}
const imports = __wbg_get_imports();
if (typeof module_or_path === 'string' || typeof Request === 'function' && module_or_path instanceof Request || typeof URL === 'function' && module_or_path instanceof URL) {
@@ -5358,7 +5358,7 @@ var OpenJPEG = (() => {
if (Module["locateFile"]) {
return locateFile("openjpeg.wasm");
}
- return new URL("openjpeg.wasm", import.meta.url).href;
+ return new URL(/* webpackIgnore: true */ "openjpeg.wasm", import.meta.url).href;
}
function getBinarySync(file) {
if (file == wasmBinaryFile && wasmBinary) {
File diff suppressed because it is too large Load Diff
@@ -1,30 +0,0 @@
diff --git a/index.js b/index.js
index 2fae107d03b30aff9320d135ec79c049c51f298a..32ec707ddbf8937e3e130c3deac1060c308bf439 100644
--- a/index.js
+++ b/index.js
@@ -1,6 +1,6 @@
'use strict';
-const {PassThrough} = require('stream');
+const {PassThrough, pipeline} = require('stream');
const extend = require('extend');
let debug = () => {};
@@ -185,7 +185,7 @@ function retryRequest(requestOpts, opts, callback) {
.on('complete', (...params) => handleFinish(params))
.on('finish', (...params) => handleFinish(params));
- requestStream.pipe(delayStream);
+ pipeline(requestStream, delayStream, () => {});
} else {
activeRequest = opts.request(requestOpts, onResponse);
}
@@ -251,7 +251,7 @@ function retryRequest(requestOpts, opts, callback) {
// No more attempts need to be made, just continue on.
if (streamMode) {
retryStream.emit('response', response);
- delayStream.pipe(retryStream);
+ pipeline(delayStream, retryStream, () => {});
requestStream.on('error', err => {
retryStream.destroy(err);
});
@@ -1,30 +0,0 @@
diff --git a/index.js b/index.js
index 298a351097d70a7fb005b6961f4d58247e391d8f..6a809ace0349d40cb2b6732ad66fd8ad208698ca 100644
--- a/index.js
+++ b/index.js
@@ -1,6 +1,6 @@
'use strict';
-const {PassThrough} = require('stream');
+const {PassThrough, pipeline} = require('stream');
const extend = require('extend');
let debug = () => {};
@@ -185,7 +185,7 @@ function retryRequest(requestOpts, opts, callback) {
.on('complete', (...params) => handleFinish(params))
.on('finish', (...params) => handleFinish(params));
- requestStream.pipe(delayStream);
+ pipeline(requestStream, delayStream, () => {});
} else {
activeRequest = opts.request(requestOpts, onResponse);
}
@@ -251,7 +251,7 @@ function retryRequest(requestOpts, opts, callback) {
// No more attempts need to be made, just continue on.
if (streamMode) {
retryStream.emit('response', response);
- delayStream.pipe(retryStream);
+ pipeline(delayStream, retryStream, () => {});
requestStream.on('error', err => {
retryStream.destroy(err);
});
@@ -1,57 +0,0 @@
diff --git a/lib/sandboxed_module.js b/lib/sandboxed_module.js
index 1cd6743..4718b97 100644
--- a/lib/sandboxed_module.js
+++ b/lib/sandboxed_module.js
@@ -4,7 +4,7 @@ var Module = require('module');
var fs = require('fs');
var vm = require('vm');
var path = require('path');
-var builtinModules = require('./builtin_modules.json');
+var builtinModules = Module.builtinModules || require('./builtin_modules.json');
var parent = module.parent;
var globalOptions = {};
var registeredBuiltInSourceTransformers = ['coffee'];
@@ -157,12 +157,20 @@ SandboxedModule.prototype._createRecursiveRequireProxy = function() {
var cache = Object.create(null);
var required = this._getRequires();
for (var key in required) {
- var injectedFilename = requireLike(this.filename).resolve(key);
- cache[injectedFilename] = required[key];
+ // Under Yarn PnP, resolution from a transitive dependency's context may fail
+ // for packages not declared in that dependency's package.json. Silently skip
+ // cache pre-population on failure; the mock will still be injected via the
+ // inject map in requireInterceptor or resolved via RecursiveRequireProxy fallback.
+ try {
+ var injectedFilename = requireLike(this.filename).resolve(key);
+ cache[injectedFilename] = required[key];
+ } catch (e) {}
}
cache[this.filename] = this.exports;
var globals = this.globals;
+ // Store the top-level module's filename for PnP fallback resolution
+ var topLevelFilename = this.filename;
var options;
if(!this._options.sourceTransformersSingleOnly && this._options.sourceTransformers){
options = {
@@ -208,8 +216,18 @@ SandboxedModule.prototype._createRecursiveRequireProxy = function() {
if (request in cache) return cache[request];
return require(request);
}
- // cached modules
- var requestedFilename = requireLike(this.filename).resolve(request);
+ // Resolve the requested module filename.
+ // Under Yarn PnP, packages can only resolve their declared dependencies.
+ // When sandboxed-module loads a transitive dependency, the resolution context
+ // may not have access to all needed packages. Fall back to resolving from
+ // the top-level module's context (the module under test).
+ var requestedFilename;
+ try {
+ requestedFilename = requireLike(this.filename).resolve(request);
+ } catch (e) {
+ if (this.filename === topLevelFilename) throw e;
+ requestedFilename = requireLike(topLevelFilename).resolve(request);
+ }
if (requestedFilename in cache) return cache[requestedFilename];
var sandboxedModule = createInnerSandboxedModule(requestedFilename)
return sandboxedModule.exports;
-57
View File
@@ -1,57 +0,0 @@
diff --git a/index.js b/index.js
index 768f8ca..a882f4d 100644
--- a/index.js
+++ b/index.js
@@ -788,29 +788,29 @@ SendStream.prototype.stream = function stream (path, options) {
// pipe
var stream = fs.createReadStream(path, options)
this.emit('stream', stream)
- stream.pipe(res)
-
- // cleanup
- function cleanup () {
- destroy(stream, true)
- }
-
- // response finished, cleanup
- onFinished(res, cleanup)
-
- // error handling
- stream.on('error', function onerror (err) {
- // clean up stream early
- cleanup()
-
- // error
- self.onStatError(err)
- })
-
- // end
- stream.on('end', function onend () {
- self.emit('end')
- })
+ Stream.pipeline(stream, res, err => { if (err) { self.onStatError(err) } else { self.emit('end') } })
+
+ // // cleanup
+ // function cleanup () {
+ // destroy(stream, true)
+ // }
+ //
+ // // response finished, cleanup
+ // onFinished(res, cleanup)
+ //
+ // // error handling
+ // stream.on('error', function onerror (err) {
+ // // clean up stream early
+ // cleanup()
+ //
+ // // error
+ // self.onStatError(err)
+ // })
+ //
+ // // end
+ // stream.on('end', function onend () {
+ // self.emit('end')
+ // })
}
/**
@@ -1,57 +0,0 @@
diff --git a/build/src/index.js b/build/src/index.js
index a101736..a87f6b9 100644
--- a/build/src/index.js
+++ b/build/src/index.js
@@ -130,6 +130,9 @@ function createMultipartStream(boundary, multipart) {
}
else {
part.body.pipe(stream, { end: false });
+ part.body.on('error', (err) => {
+ stream.destroy(err);
+ });
part.body.on('end', () => {
stream.write('\r\n');
stream.write(finale);
@@ -184,25 +187,25 @@ function teenyRequest(reqOpts, callback) {
// Stream mode
const requestStream = streamEvents(new stream_1.PassThrough());
// eslint-disable-next-line @typescript-eslint/no-explicit-any
- let responseStream;
- requestStream.once('reading', () => {
- if (responseStream) {
- (0, stream_1.pipeline)(responseStream, requestStream, () => { });
- }
- else {
- requestStream.once('response', () => {
- (0, stream_1.pipeline)(responseStream, requestStream, () => { });
- });
- }
- });
+ // let responseStream;
+ // requestStream.once('reading', () => {
+ // if (responseStream) {
+ // (0, stream_1.pipeline)(responseStream, requestStream, () => { });
+ // }
+ // else {
+ // requestStream.once('response', () => {
+ // (0, stream_1.pipeline)(responseStream, requestStream, () => { });
+ // });
+ // }
+ // });
options.compress = false;
teenyRequest.stats.requestStarting();
- fetch(uri, options).then(res => {
+ (0, node_fetch_1.default)(uri, options).then(res => {
- teenyRequest.stats.requestFinished();
- responseStream = res.body;
- responseStream.on('error', (err) => {
- requestStream.emit('error', err);
- });
+ teenyRequest.stats.requestFinished(); (0, stream_1.pipeline)(res.body, requestStream, () => {});
+ // responseStream = res.body;
+ // responseStream.on('error', (err) => {
+ // requestStream.emit('error', err);
+ // });
const response = fetchToRequestResponse(options, res);
requestStream.emit('response', response);
}, err => {
@@ -1,58 +0,0 @@
diff --git a/build/src/index.js b/build/src/index.js
index af5d15e260e2a47588c7c536447fe84bd3f86136..2b63d0c0b1eb6595c7a0bb314c1df792454c1a72 100644
--- a/build/src/index.js
+++ b/build/src/index.js
@@ -115,6 +115,9 @@ function createMultipartStream(boundary, multipart) {
}
else {
part.body.pipe(stream, { end: false });
+ part.body.on('error', (err) => {
+ stream.destroy(err);
+ });
part.body.on('end', () => {
stream.write('\r\n');
stream.write(finale);
@@ -168,25 +171,27 @@ function teenyRequest(reqOpts, callback) {
// Stream mode
const requestStream = streamEvents(new stream_1.PassThrough());
// eslint-disable-next-line @typescript-eslint/no-explicit-any
- let responseStream;
- requestStream.once('reading', () => {
- if (responseStream) {
- (0, stream_1.pipeline)(responseStream, requestStream, () => { });
- }
- else {
- requestStream.once('response', () => {
- (0, stream_1.pipeline)(responseStream, requestStream, () => { });
- });
- }
- });
+ // let responseStream;
+ // requestStream.once('reading', () => {
+ // if (responseStream) {
+ // (0, stream_1.pipeline)(responseStream, requestStream, () => { });
+ // }
+ // else {
+ // requestStream.once('response', () => {
+ // (0, stream_1.pipeline)(responseStream, requestStream, () => { });
+ // });
+ // }
+ // });
+
+
options.compress = false;
teenyRequest.stats.requestStarting();
(0, node_fetch_1.default)(uri, options).then(res => {
- teenyRequest.stats.requestFinished();
- responseStream = res.body;
- responseStream.on('error', (err) => {
- requestStream.emit('error', err);
- });
+ teenyRequest.stats.requestFinished(); stream_1.pipeline(res.body, requestStream, () => {});
+ // responseStream = res.body;
+ // responseStream.on('error', (err) => {
+ // requestStream.emit('error', err);
+ // });
const response = fetchToRequestResponse(options, res);
requestStream.emit('response', response);
}, err => {
@@ -1,81 +0,0 @@
diff --git a/dist/WorkerPool.js b/dist/WorkerPool.js
index 4145779f08eefafd0c18394806b6409c595ac5bb..aa16dd6a0f463804455164493a1e75e80cdf656a 100644
--- a/dist/WorkerPool.js
+++ b/dist/WorkerPool.js
@@ -258,6 +258,19 @@ class PoolWorker {
finalCallback();
break;
}
+ case 'logMessage':
+ {
+ const {
+ data: { loggerName, methodName, args }
+ } = message;
+ const {
+ data: jobData
+ } = this.jobs[id];
+ const logger = jobData.getLogger(loggerName);
+ logger[methodName].apply(logger, args);
+ finalCallback();
+ break;
+ }
case 'emitWarning':
{
const {
diff --git a/dist/index.js b/dist/index.js
index 75cd30fb63dc864057c1afc866f43fc7cc0a8020..d834af6ce1e2cd4473c4ae1b325d63eb1190c17e 100644
--- a/dist/index.js
+++ b/dist/index.js
@@ -43,6 +43,7 @@ function pitch() {
sourceMap: this.sourceMap,
emitError: this.emitError,
emitWarning: this.emitWarning,
+ getLogger: this.getLogger,
loadModule: this.loadModule,
resolve: this.resolve,
getResolve: this.getResolve,
diff --git a/dist/worker.js b/dist/worker.js
index 8e67959e4b7c9fd0db116509b1459636ba13a097..aca94f1442906baf179ada8e6a56501bbf71c480 100644
--- a/dist/worker.js
+++ b/dist/worker.js
@@ -90,6 +90,22 @@ function writeJson(data) {
writePipeWrite(lengthBuffer);
writePipeWrite(messageBuffer);
}
+const LOGGER_METHODS = ['error', 'warn', 'info', 'log', 'debug', 'trace', 'group', 'groupEnd', 'groupCollapsed', 'status', 'clear', 'profile', 'profileEnd'];
+class Logger {
+ constructor(id, loggerName) {
+ this.id = id
+ this.loggerName = loggerName
+ for (const methodName of LOGGER_METHODS) {
+ this[methodName] = (...args) => {
+ writeJson({
+ type: 'logMessage',
+ id: this.id,
+ data: { loggerName, methodName, args }
+ })
+ }
+ }
+ }
+}
const queue = (0, _queue.default)(({
id,
data
@@ -190,6 +206,7 @@ const queue = (0, _queue.default)(({
}
return options;
},
+ getLogger: (name) => new Logger(id, name),
emitWarning: warning => {
writeJson({
type: 'emitWarning',
@@ -211,6 +228,9 @@ const queue = (0, _queue.default)(({
module._compile(code, filename); // eslint-disable-line no-underscore-dangle
return module.exports;
},
+ addDependency: filename => {
+ buildDependencies.push(filename);
+ },
addBuildDependency: filename => {
buildDependencies.push(filename);
},
-25
View File
@@ -1,25 +0,0 @@
approvedGitRepositories:
- "**"
enableGlobalCache: false
enableScripts: true
enableTelemetry: false
nodeLinker: node-modules
npmMinimalAgeGate: 3d
supportedArchitectures:
cpu:
- current
- arm64
- x64
libc:
- current
- glibc
os:
- current
- darwin
- linux
+28 -15
View File
@@ -1,35 +1,48 @@
# Contributing to Overleaf
Contributing to ShareLaTeX
==========================
Thank you for reading this! If you'd like to report a bug or join in the development
of Overleaf, then here are some notes on how to do that.
of ShareLaTeX, then here are some notes on how to do that.
## Reporting bugs and opening issues
*Note that ShareLaTeX is actually made up of many separate repositories (a list is available
[here](https://github.com/sharelatex/sharelatex/blob/master/README.md#other-repositories)).*
If you'd like to report a bug or open an issue, please **[check if there is an existing issue](https://github.com/overleaf/overleaf/issues).**
If there is then please add any more information that you have, or give it a 👍.
Reporting bugs and opening issues
---------------------------------
If you'd like to report a bug or open an issue then please:
1. **Find the correct repository.** ShareLaTeX is split across multiple different repositories, each containing a different service (you can find a list of [all repositories here](https://github.com/sharelatex/sharelatex/blob/master/README.md#other-repositories)). If you know the bug only applies to one service, then please open an issue in that repository. For general bugs and issues that span more than one service, please open an issue in the [sharelatex/sharelatex](https://github.com/sharelatex/sharelatex) repository.
2. **Check if there is an existing issue.** If there is then please add
any more information that you have, or give it a 👍.
When submitting an issue please describe the issue as clearly as possible, including how to
reproduce the bug, which situations it appears in, what you expected to happen, and what actually happens.
If you can include a screenshot for front end issues that is very helpful.
**Note**: If you are using [www.overleaf.com](www.overleaf.com) and have a problem, or if you would like to request a new feature, please contact the Support team at support@overleaf.com. Raise an issue here only to report bugs in the Community Edition release of Overleaf.
Pull Requests
-------------
## Pull Requests
See [our wiki](https://github.com/overleaf/overleaf/wiki)
for how to manage the Overleaf development environment and for our developer guidelines.
See [our wiki](https://github.com/sharelatex/sharelatex/wiki)
for how to manage the ShareLaTeX development environment and for our developer guidelines.
We love pull requests, so be bold with them! Don't be afraid of going ahead
and changing something, or adding a new feature. We're very happy to work with you
to get your changes merged into Overleaf.
to get your changes merged into ShareLaTeX.
If you're looking for something to work on, have a look at the [open issues](https://github.com/overleaf/overleaf/issues).
If you're looking for something to work on, have a look at the open issues in any of the repositories listed [here](https://github.com/sharelatex/sharelatex/blob/master/README.md#other-repositories).
## Security
Security
--------
Please see [our security policy](https://github.com/overleaf/overleaf/security/policy) if you would like to report a potential security vulnerability.
Please do not publish security vulnerabilities publicly until we've had a chance
to address them. All security related issues/patches should be sent directly to
security@overleaf.com where we will attempt to address them quickly. If you're
unsure whether something is a security issue or not, then please be cautious and
contact us at security@overleaf.com first.
## Contributor License Agreement
Contributor License Agreement
-----------------------------
Before we can accept any contributions of code, we need you to agree to our
[Contributor License Agreement](https://docs.google.com/forms/d/e/1FAIpQLSef79XH3mb7yIiMzZw-yALEegS-wyFetvjTiNBfZvf_IHD2KA/viewform?usp=sf_link).
+49 -268
View File
@@ -1,296 +1,77 @@
<h1 align="center">
<br>
<a href="https://www.overleaf.com"><img src="doc/logo.png" alt="Overleaf" width="300"></a>
</h1>
<h4 align="center">An open-source online real-time collaborative LaTeX editor.</h4>
<p align="center">
<img src="services/web/public/img/ol-brand/verso-logo.svg" alt="Verso" width="440">
<a href="#key-features">Key Features</a> •
<a href="https://github.com/overleaf/overleaf/wiki">Wiki</a> •
<a href="https://www.overleaf.com/for/enterprises">Server Pro</a> •
<a href="#contributing">Contributing</a> •
<a href="https://mailchi.mp/overleaf.com/community-edition-and-server-pro">Mailing List</a> •
<a href="#authors">Authors</a> •
<a href="#license">License</a>
</p>
**A collaborative, real-time editor for LaTeX, Quarto and Typst — self-hosted.**
<a href="https://www.overleaf.com"><img src="doc/screenshot.png" alt="Overleaf" ></a>
<p align="center">
Figure 1: A screenshot of Overleaf Server Pro's comments and tracked changes features.
</p>
---
## Key Features
## What is Verso?
[Overleaf](https://www.overleaf.com) is an open-source online real-time collaborative LaTeX editor. We run a hosted version at [www.overleaf.com](https://www.overleaf.com), but you can also run your own local version, and contribute to the development of Overleaf.
Verso is a fork of [Overleaf](https://github.com/overleaf/overleaf) that extends its
collaborative editing infrastructure to support [Quarto](https://quarto.org) and
[Typst](https://typst.app) projects alongside LaTeX. Think of it as Overleaf, but
not limited to LaTeX.
*[If you want help installing and maintaining Overleaf in your lab or workplace, we offer an officially supported version called Overleaf Server Pro. It also comes with extra security and admin features. Click here to find out more!](https://www.overleaf.com/for/enterprises)*
### Verso vs Overleaf
## Keeping up to date
[Overleaf](https://www.overleaf.com) is the gold standard for collaborative LaTeX
editing. Verso keeps everything that makes Overleaf great — real-time co-editing,
operational-transformation history, auth, project management, file storage — and
adds:
Sign up to the [mailing list](https://mailchi.mp/overleaf.com/community-edition-and-server-pro) to get updates on Overleaf Releases and development
- **Quarto and Typst compilers** running alongside TeX Live, dispatched
automatically from the root file's extension (`.qmd` → Quarto, `.typ` → Typst,
`.tex``latexmk`).
- **Language-aware editor** for Quarto and Typst (syntax highlighting, completions,
document outline) — not just LaTeX.
- **Publish & share compiled output** (`/p/:token` with tiered access links) — a
feature absent from Overleaf Community Edition.
- **Lumière theme** — a redesigned project dashboard and editor chrome with a
card-based grid, thumbnails, and a teal gradient identity.
- **Full i18n** — French, German, Italian, and Spanish UI translations on top of
Overleaf's English base.
- Completely **free and self-hosted**; no Overleaf subscription required.
## Installation
### Verso vs Quarto
We have detailed installation instructions in our wiki:
[Quarto](https://quarto.org) is a command-line tool: you install it locally, write
`.qmd` files in any text editor, and run `quarto render` in a terminal. It is
excellent for solo authors with full control over their environment.
* [Overleaf Quick Start Guide](https://github.com/overleaf/overleaf/wiki/Quick-Start-Guide)
Verso wraps Quarto in a collaborative web editor:
## Upgrading
- **No local install** — Quarto, Typst, TeX Live and Python run on the server.
- **Real-time collaboration** — multiple people edit the same `.qmd` simultaneously
with live cursors and conflict-free merging.
- **Not just Quarto** — LaTeX and Typst projects live in the same workspace, under
the same auth and history system.
- **Publish in one click** — RevealJS decks and PDFs are served at a stable link
without leaving the browser.
If you are upgrading from a previous version of Overleaf, please see the [Release Notes section on the Wiki](https://github.com/overleaf/overleaf/wiki/Home) for all of the versions between your current version and the version you are upgrading to.
Verso is not a replacement for Quarto's CLI — it is a platform that makes Quarto
accessible as a shared, always-on service.
## Overleaf Docker Image
### Verso vs Typst.app
This repo contains two dockerfiles, `Dockerfile-base`, which builds the
`sharelatex/sharelatex-base` image, and `Dockerfile` which builds the
`sharelatex/sharelatex` (or "community") image.
[Typst.app](https://typst.app) is a cloud-hosted web editor for Typst. It is
polished and fast, but it is a proprietary SaaS product and only supports Typst.
The Base image generally contains the basic dependencies like `wget` and
`aspell`, plus `texlive`. We split this out because it's a pretty heavy set of
dependencies, and it's nice to not have to rebuild all of that every time.
Verso differs in that:
The `sharelatex/sharelatex` image extends the base image and adds the actual Overleaf code
and services.
- It is **self-hosted** and open-source (AGPL v3) — you control your data.
- It supports **three languages** (Typst, LaTeX, Quarto) in one instance.
- Real-time collaboration is powered by **operational transformation** (the same
engine as Overleaf), not CRDTs, which means it handles concurrent edits
gracefully for long documents.
- It ships with a full **project history** and version-restore workflow.
Use `make build-base` and `make build-community` from `server-ce/` to build these images.
If you only need Typst and want a lighter, Typst-focused alternative, have a look
at **[Collabst](https://github.com/herluf-ba/collabst)** — an open-source,
self-hosted collaborative Typst editor that is independent of the Overleaf
codebase and shows a lot of promise.
We use the [Phusion base-image](https://github.com/phusion/baseimage-docker)
(which is extended by our `base` image) to provide us with a VM-like container
in which to run the Overleaf services. Baseimage uses the `runit` service
manager to manage services, and we add our init-scripts from the `server-ce/runit`
folder.
---
## Features
## Contributing
- **Real-time collaboration** — multiple editors, live cursors, full project history and version restore.
- **Three compilers, auto-dispatched** by root file extension:
| Root file | Compiler | Typical output |
|-----------|----------|----------------|
| `.qmd` | Quarto | PDF (via Typst or LaTeX), HTML, or RevealJS |
| `.tex` | `latexmk` / TeX Live | PDF |
| `.typ` | Typst | PDF |
- **Language-aware editor for all three** — syntax highlighting, completions, and a document outline panel for LaTeX, Quarto and Typst.
- **Format badge** on the project dashboard; compiler dropdown greys out inapplicable engines.
- **Publish & share** — compile and snapshot to `/p/:token` with three independent access tiers (project members / any logged-in user / public). HTML/RevealJS decks are served live; PDFs are embedded inline. A **Present** toolbar button links directly to the published deck.
- **RevealJS thumbnails** — the first slide of a presentation is rendered as a preview card in the project list.
- **Quarto Python cells** — optional per-project virtual environment built from `requirements.txt`, so Python code chunks execute during render.
- **Visual formatting toolbar** — bold, italic, headings and inline code shortcuts for Quarto (`.qmd`) and Typst (`.typ`) files, in addition to Overleaf's existing LaTeX toolbar.
- **Lumière theme** — card-based project dashboard with PDF/slide thumbnails, a teal gradient identity, dark editor chrome, and an XS compact list view.
- **i18n** — French, German, Italian and Spanish UI translations.
- **Auto-compile** — preview refreshes automatically after you stop typing.
Please see the [CONTRIBUTING](https://github.com/overleaf/overleaf/blob/master/CONTRIBUTING.md) file for information on contributing to the development of Overleaf. See [our wiki](https://github.com/overleaf/overleaf/wiki/Developer-Guidelines) for information on setting up a development environment and how to recompile and run Overleaf after modifications.
---
## Authors
## Releases
### Alpha 1
The initial public release. Established Verso as an Overleaf fork with first-class
multi-language support:
- Quarto (`.qmd`) and Typst (`.typ`) compilers running alongside TeX Live,
dispatched automatically by root file extension — no per-project configuration.
- Language-aware editor for Quarto: Markdown highlighting, code-chunk completions
(`{python}`, `{r}`, `{julia}`, `{ojs}`…), callout and fenced-div completions,
cross-reference completions (`@fig-`, `@tbl-`, `@sec-`…).
- Language-aware editor for Typst: syntax highlighting and completions for
functions, imports, math and markup.
- Document outline panel for all three languages (LaTeX `\section`, Quarto `#`,
Typst `=`).
- Format badge on the project dashboard; compiler selector greys out inapplicable
engines for the current root file.
- Publish & share compiled output — HTML/RevealJS decks and PDFs hosted at
`/p/:token` with tiered access links (project / logged-in / public), each
independently resettable.
- Quarto Python code-cell execution via an optional per-project `requirements.txt`
virtual environment.
- Verso branding: name, logo and Kubernetes production deploy workflow.
### Alpha 2
Refinements to the Typst editor and the format badge system:
- **Quarto format sub-types** — the project badge now distinguishes *Quarto PDF*
from *Quarto Slides*, reading the frontmatter `format:` to pick the right label.
- **Python packages for collaborators** — Quarto Python package installation
extended to all users who have write access to the project, not only the owner.
- **Typst syntax highlighting overhaul** — complete grammar rewrite covering:
function calls and named argument keys, multi-line display math, `#{…}` code
blocks, content blocks, `show`-rule bodies, `let`-value bindings, and keyword
vs identifier disambiguation.
- **Typst visual formatting** — bold and italic toolbar buttons and keyboard
shortcuts (`Ctrl+B`, `Ctrl+I`), plus underline, small-caps and hyperlink
buttons, matching the Quarto and LaTeX toolbar experience.
### Alpha 3
- **Lumière theme** — redesigned project dashboard with a card grid, PDF/slide
thumbnails, parallax hover effects, a teal gradient identity and a dark editor
chrome. Includes an XS compact list view and a tile zoom slider.
- **Full i18n** — French, German, Italian and Spanish translations covering the
complete UI (login, dashboard, editor, settings, emails).
- **Visual editors for Quarto and Typst** — bold, italic, headings and inline code
shortcuts in the toolbar for `.qmd` and `.typ` files.
- **Top/bottom split view** — new editor layout that stacks the source editor
above the PDF preview vertically, in addition to the existing side-by-side mode.
- **Bidirectional format export** — LaTeX projects can be converted to Typst and
Typst projects to LaTeX via pandoc. Available from the File menu in the editor.
- **Mobile layout** — project dashboard, search bar, footer and editor all
adapted for phone screen sizes.
---
## Known issues
- **Large file upload timeouts** — uploads of large files on slow connections
can time out at the proxy layer. A streaming response fix is pending.
---
## Security model — trusted environments only
> [!CAUTION]
> Verso is designed for **closed groups of trusted users** (a lab, a class, a
> small team). All three compilers can execute arbitrary code on the server:
>
> - LaTeX with shell-escape enabled can run system commands.
> - Quarto Python cells execute Python code directly.
> - Typst's scripting layer is sandboxed by design, but runs server-side.
>
> There is **no per-project sandbox or resource isolation** beyond what the
> operating system provides. Exposing Verso to the public internet with open
> registration is not recommended. If you need to host a collaborative
> LaTeX editor for untrusted users or at scale, look at
> [Overleaf's non-Community offerings](https://www.overleaf.com/for/enterprises),
> which include proper sandboxing and enterprise access controls.
---
## Quick start
### With Docker
```bash
docker run -d \
-p 80:80 \
-v ~/verso_data:/var/lib/overleaf \
--name verso \
registry.alocoq.fr/verso:latest
```
Open `http://localhost`, then visit `/launchpad` on first run to create the admin
account.
### Build from source
```bash
cd server-ce
make build-base # base OS image: system deps, Quarto, Typst, TeX Live
make build-community # application image: Node services + compiled frontend
```
| File | Purpose |
|------|---------|
| `server-ce/Dockerfile-base` | Base image — system deps, Quarto (with Typst) and TeX Live |
| `server-ce/Dockerfile` | App image — Node services and the compiled React frontend |
---
## Architecture
Verso is a microservices monorepo (Yarn workspaces). All services run inside a
single container managed by `runit`, with `nginx` as the front router.
```
browser ──→ nginx:80
├── / ──────────────────→ web:4000 (main app, React UI)
├── /socket.io ──────────→ real-time:3026 (WebSocket, OT engine)
├── /p/:token ───────────→ web (published output)
└── /project/*/output/* → clsi-nginx:8080 (compiled output files)
web → document-updater → Redis pub/sub → real-time → browser
web → CLSI (quarto render / latexmk / typst) → output files → nginx → browser
```
| Service | Role |
|---------|------|
| `web` | HTTP API, React frontend, auth, project & sharing management |
| `real-time` | WebSocket layer, live cursors and edit sync |
| `document-updater` | Operational transformation, Redis pub/sub |
| `clsi` | Compiler — runs `quarto render`, `latexmk` or `typst` and serves output |
| `docstore` | Document text storage (MongoDB) |
| `filestore` | Binary file storage (S3 or local) |
| `project-history` | Change history and version tracking |
---
## Environment variables
Verso inherits all of Overleaf's environment variables (prefixed `OVERLEAF_`).
The most commonly needed:
| Variable | Default | Description |
|----------|---------|-------------|
| `OVERLEAF_APP_NAME` | `Verso` | Name shown in the UI |
| `OVERLEAF_NAV_TITLE` | — | Instance name/version shown in the top bar |
| `OVERLEAF_MONGO_URL` | `mongodb://mongo/sharelatex` | MongoDB connection string |
| `OVERLEAF_REDIS_HOST` | `localhost` | Redis host |
| `OVERLEAF_SITE_URL` | — | Public URL (used in emails and published links) |
| `OVERLEAF_SITE_LANGUAGE` | `en` | Default UI language (e.g. `fr`) |
| `OVERLEAF_ENABLE_PROJECT_PYTHON_VENV` | `false` | Allow Quarto Python cells to use a project `requirements.txt` |
| `OVERLEAF_ADMIN_EMAIL` | — | Email shown on the launchpad for the first admin account |
See the [Overleaf Server documentation](https://github.com/overleaf/overleaf/wiki)
for the full list.
---
## Relation to Overleaf
Verso is a fork of [Overleaf Community Edition](https://github.com/overleaf/overleaf).
Everything that Overleaf CE provides — real-time collaboration, operational-transformation
history, auth, project management, binary file storage — is inherited unchanged. The
Verso-specific additions are listed in the Features section and tracked across releases above.
Verso is not affiliated with Overleaf Ltd.
---
## Supporting the ecosystem
Verso is not accepting contributions or donations at this time. If you find it
useful and want to support the broader ecosystem it builds on:
- **Support Overleaf** — Verso is built on Overleaf's infrastructure. The best
way to support their work is to use or subscribe to
[Overleaf](https://www.overleaf.com) and encourage your institution to do the
same.
- **Support Typst** — [Typst GmbH](https://typst.app) is the company behind the
Typst compiler. Using Typst.app or sponsoring the
[Typst project on GitHub](https://github.com/typst/typst) helps sustain the
language itself.
- **Support RevealJS** — Verso uses [Reveal.js](https://revealjs.com) for
HTML presentations. Consider sponsoring the
[RevealJS project on GitHub](https://github.com/hakimel/reveal.js).
---
[The Overleaf Team](https://www.overleaf.com/about)
## License
GNU Affero General Public License v3 — see [LICENSE](LICENSE).
The code in this repository is released under the GNU AFFERO GENERAL PUBLIC LICENSE, version 3. A copy can be found in the `LICENSE` file.
Copyright © Overleaf, 20142026 (original code).
Verso modifications © Aloïs Coquillard, 2026.
Copyright (c) Overleaf, 2014-2021.
-50
View File
@@ -1,50 +0,0 @@
# Verso — Next Alpha Roadmap
Ideas and features deferred from the current alpha.
---
## Next alpha (post-current)
### Typst editing experience (inspired by Collabst)
- **typst.ts WASM preview** — Run the Typst compiler in the browser via
WebAssembly (typst.ts). This would give instant, sub-second preview
without a server round-trip, and would eliminate the entire class of
race conditions in the CLSI watcher (files written → typst compiles →
resolver missed). Could coexist with the CLSI watcher for PDF export
while using the WASM path for live preview.
- **Tinymist LSP integration** — Wire up
[Tinymist](https://github.com/Myriad-Dreamin/tinymist) (the Typst
language server) behind a WebSocket proxy. Would give Typst files
first-class autocomplete, hover docs, go-to-definition, and inline
error diagnostics — the main editing comfort gap vs. a native editor.
### Editor UX for non-LaTeX formats (.typ, .qmd, .md)
- **Visual/rich-text editing mode** — A toggle between raw source and a
rendered-in-place view for `.typ`, `.qmd`, and `.md` files (similar to
Overleaf's rich-text mode for LaTeX). Users who don't know Typst or
Markdown syntax should be able to edit content without seeing markup.
CodeMirror 6 already supports this pattern via a custom `NodeView` layer
or a separate Prosemirror bridge.
- **Toolbar / insertion shortcuts** — A formatting toolbar and keyboard
shortcuts for common operations, adapted per file type:
- **All formats**: bold, italic, underline, headings, bullet/numbered
lists, inline code, links.
- **Quarto / Markdown**: insert image, insert table, insert code block
with language tag.
- **Quarto RevealJS**: insert slide divider (`---`), insert speaker
notes (`::: notes`), insert columns layout, insert video embed
(using Quarto's `{{< video >}}` shortcode).
### AI writing assistant
- **In-editor AI assistant** — Inline writing help similar to what Overleaf
and CoCalc offer: suggest completions, rephrase selections, explain LaTeX
errors, and generate boilerplate (figures, tables, equations). Should work
across all three formats (`.tex`, `.typ`, `.qmd`). Backend would proxy
requests to a configurable model API (Claude, OpenAI-compatible) so
self-hosters can bring their own key.
-3
View File
@@ -1,3 +0,0 @@
/* eslint-disable no-undef */
rs.initiate({ _id: 'overleaf', members: [{ _id: 0, host: 'mongo:27017' }] })
-3
View File
@@ -1,3 +0,0 @@
/compiles/*
!.gitkeep
.env
-76
View File
@@ -1,76 +0,0 @@
# Overleaf Community Edition, development environment
## Building and running
In this `develop` directory, build the services:
```shell
bin/build
```
> [!NOTE]
> If Docker is running out of RAM while building the services in parallel, create a `.env` file in this directory containing `COMPOSE_PARALLEL_LIMIT=1`.
Then start the services:
```shell
bin/up
```
Once the services are running, open <http://localhost/launchpad> to create the first admin account.
## Development
To avoid running `bin/build && bin/up` after every code change, you can run Overleaf
Community Edition in _development mode_, where services will automatically update on code changes.
To do this, use the included `bin/dev` script:
```shell
bin/dev
```
This will start all services using `node --watch`, which will automatically monitor the code and restart the services as necessary.
To improve performance, you can start only a subset of the services in development mode by providing a space-separated list to the `bin/dev` script:
```shell
bin/dev [service1] [service2] ... [serviceN]
```
> [!NOTE]
> Starting the `web` service in _development mode_ will only update the `web`
> service when backend code changes. In order to automatically update frontend
> code as well, make sure to start the `webpack` service in _development mode_
> as well.
If no services are named, all services will start in development mode.
## Debugging
When run in _development mode_ most services expose a debugging port to which
you can attach a debugger such as
[the inspector in Chrome's Dev Tools](chrome://inspect/) or one integrated into
an IDE. The following table shows the port exposed on the **host machine** for
each service:
| Service | Port |
| ------------------ | ---- |
| `web` | 9229 |
| `clsi` | 9230 |
| `chat` | 9231 |
| `docstore` | 9233 |
| `document-updater` | 9234 |
| `filestore` | 9235 |
| `notifications` | 9236 |
| `real-time` | 9237 |
| `history-v1` | 9239 |
| `project-history` | 9240 |
To attach to a service using Chrome's _remote debugging_, go to
<chrome://inspect/> and make sure _Discover network targets_ is checked. Next
click _Configure..._ and add an entry `localhost:[service port]` for each of the
services you want to attach a debugger to.
After adding an entry, the service will show up as a _Remote Target_ that you
can inspect and debug.
-3
View File
@@ -1,3 +0,0 @@
#!/usr/bin/env bash
docker compose build --pull "$@"
-3
View File
@@ -1,3 +0,0 @@
#!/usr/bin/env bash
docker-compose -f docker-compose.yml -f docker-compose.dev.yml up --no-deps --detach "$@"
-3
View File
@@ -1,3 +0,0 @@
#!/usr/bin/env bash
docker compose down "$@"
-9
View File
@@ -1,9 +0,0 @@
#!/usr/bin/env bash
docker compose logs --follow --tail 10 --no-color "$@" \
| ggrep --line-buffered --invert-match "global.gc" \
| ggrep --line-buffered --invert-match "health.check" \
| ggrep --line-buffered --invert-match "slow event loop" \
| ggrep --line-buffered --invert-match "process.memoryUsage" \
| ggrep --line-buffered --only-matching "[{].*" \
| bunyan --output short
-3
View File
@@ -1,3 +0,0 @@
#!/usr/bin/env bash
docker compose exec -it "$@" /bin/bash
-3
View File
@@ -1,3 +0,0 @@
#!/usr/bin/env bash
docker compose up --detach "$@"
View File
-24
View File
@@ -1,24 +0,0 @@
CHAT_HOST=chat
CLSI_HOST=clsi
DOWNLOAD_HOST=clsi-nginx
DOCSTORE_HOST=docstore
DOCUMENT_UPDATER_HOST=document-updater
FILESTORE_HOST=filestore
GRACEFUL_SHUTDOWN_DELAY_SECONDS=0
HISTORY_V1_HOST=history-v1
HISTORY_REDIS_HOST=redis
LISTEN_ADDRESS=0.0.0.0
MONGO_HOST=mongo
MONGO_URL=mongodb://mongo/sharelatex?directConnection=true
NOTIFICATIONS_HOST=notifications
PROJECT_HISTORY_HOST=project-history
QUEUES_REDIS_HOST=redis
DSMP_REDIS_HOST=redis
REALTIME_HOST=real-time
REDIS_HOST=redis
SESSION_SECRET=foo
V1_HISTORY_HOST=history-v1
WEBPACK_HOST=webpack
WEB_API_PASSWORD=overleaf
WEB_API_USER=overleaf
WEB_HOST=web
-128
View File
@@ -1,128 +0,0 @@
services:
clsi:
command: ["node", "--watch", "app.js"]
environment:
- NODE_OPTIONS=--inspect=0.0.0.0:9229
ports:
- "127.0.0.1:9230:9229"
volumes:
- ../services/clsi/app:/overleaf/services/clsi/app
- ../services/clsi/app.js:/overleaf/services/clsi/app.js
- ../services/clsi/config:/overleaf/services/clsi/config
chat:
command: ["node", "--watch", "app.js"]
environment:
- NODE_OPTIONS=--inspect=0.0.0.0:9229
ports:
- "127.0.0.1:9231:9229"
volumes:
- ../services/chat/app:/overleaf/services/chat/app
- ../services/chat/app.js:/overleaf/services/chat/app.js
- ../services/chat/config:/overleaf/services/chat/config
docstore:
command: ["node", "--watch", "app.js"]
environment:
- NODE_OPTIONS=--inspect=0.0.0.0:9229
ports:
- "127.0.0.1:9233:9229"
volumes:
- ../services/docstore/app:/overleaf/services/docstore/app
- ../services/docstore/app.js:/overleaf/services/docstore/app.js
- ../services/docstore/config:/overleaf/services/docstore/config
document-updater:
command: ["node", "--watch", "app.js"]
environment:
- NODE_OPTIONS=--inspect=0.0.0.0:9229
ports:
- "127.0.0.1:9234:9229"
volumes:
- ../services/document-updater/app:/overleaf/services/document-updater/app
- ../services/document-updater/app.js:/overleaf/services/document-updater/app.js
- ../services/document-updater/config:/overleaf/services/document-updater/config
filestore:
command: ["node", "--watch", "app.js"]
environment:
- NODE_OPTIONS=--inspect=0.0.0.0:9229
ports:
- "127.0.0.1:9235:9229"
volumes:
- ../services/filestore/app:/overleaf/services/filestore/app
- ../services/filestore/app.js:/overleaf/services/filestore/app.js
- ../services/filestore/config:/overleaf/services/filestore/config
history-v1:
command: ["node", "--watch", "app.js"]
environment:
- NODE_OPTIONS=--inspect=0.0.0.0:9229
ports:
- "127.0.0.1:9239:9229"
volumes:
- ../services/history-v1/api:/overleaf/services/history-v1/api
- ../services/history-v1/app.js:/overleaf/services/history-v1/app.js
- ../services/history-v1/config:/overleaf/services/history-v1/config
- ../services/history-v1/storage:/overleaf/services/history-v1/storage
- ../services/history-v1/knexfile.js:/overleaf/services/history-v1/knexfile.js
- ../services/history-v1/migrations:/overleaf/services/history-v1/migrations
notifications:
command: ["node", "--watch", "app.ts"]
environment:
- NODE_OPTIONS=--inspect=0.0.0.0:9229
ports:
- "127.0.0.1:9236:9229"
volumes:
- ../services/notifications/app:/overleaf/services/notifications/app
- ../services/notifications/app.ts:/overleaf/services/notifications/app.ts
- ../services/notifications/config:/overleaf/services/notifications/config
project-history:
command: ["node", "--watch", "app.js"]
environment:
- NODE_OPTIONS=--inspect=0.0.0.0:9229
ports:
- "127.0.0.1:9240:9229"
volumes:
- ../services/project-history/app:/overleaf/services/project-history/app
- ../services/project-history/app.js:/overleaf/services/project-history/app.js
- ../services/project-history/config:/overleaf/services/project-history/config
real-time:
command: ["node", "--watch", "app.js"]
environment:
- NODE_OPTIONS=--inspect=0.0.0.0:9229
ports:
- "127.0.0.1:9237:9229"
volumes:
- ../services/real-time/app:/overleaf/services/real-time/app
- ../services/real-time/app.js:/overleaf/services/real-time/app.js
- ../services/real-time/config:/overleaf/services/real-time/config
web:
command: ["node", "--watch", "app.mjs", "--watch-locales"]
environment:
- NODE_OPTIONS=--inspect=0.0.0.0:9229
ports:
- "127.0.0.1:9229:9229"
volumes:
- ../services/web/app:/overleaf/services/web/app
- ../services/web/app.mjs:/overleaf/services/web/app.mjs
- ../services/web/config:/overleaf/services/web/config
- ../services/web/locales:/overleaf/services/web/locales
- ../services/web/modules:/overleaf/services/web/modules
- ../services/web/public:/overleaf/services/web/public
webpack:
volumes:
- ../services/web/app:/overleaf/services/web/app
- ../services/web/config:/overleaf/services/web/config
- ../services/web/frontend:/overleaf/services/web/frontend
- ../services/web/locales:/overleaf/services/web/locales
- ../services/web/modules:/overleaf/services/web/modules
- ../services/web/public:/overleaf/services/web/public
- ../services/web/transform:/overleaf/services/web/transform
- ../services/web/types:/overleaf/services/web/types
- ../services/web/webpack-plugins:/overleaf/services/web/webpack-plugins
-175
View File
@@ -1,175 +0,0 @@
volumes:
clsi-cache:
filestore-public-files:
filestore-template-files:
filestore-uploads:
filestore-user-files:
mongo-data:
redis-data:
sharelatex-data:
web-data:
history-v1-buckets:
services:
chat:
build:
context: ..
dockerfile: services/chat/Dockerfile
env_file:
- dev.env
clsi:
build:
context: ..
dockerfile: services/clsi/Dockerfile
target: with-texlive
env_file:
- dev.env
environment:
- SANDBOXED_COMPILES=false
user: root
volumes:
- ${PWD}/compiles:/overleaf/services/clsi/compiles
- ${PWD}/output:/overleaf/services/clsi/output
- ${DOCKER_SOCKET_PATH:-/var/run/docker.sock}:/var/run/docker.sock
- clsi-cache:/overleaf/services/clsi/cache
clsi-nginx:
image: nginx:1.28
read_only: true
tmpfs:
- /tmp
- /var/cache/nginx
- /run
volumes:
- ${PWD}/output:/output:ro
- ../services/clsi/nginx.conf:/etc/nginx/conf.d/nginx.conf:ro
docstore:
build:
context: ..
dockerfile: services/docstore/Dockerfile
env_file:
- dev.env
document-updater:
build:
context: ..
dockerfile: services/document-updater/Dockerfile
env_file:
- dev.env
filestore:
build:
context: ..
dockerfile: services/filestore/Dockerfile
env_file:
- dev.env
# environment:
# - ENABLE_CONVERSIONS=true
volumes:
- filestore-public-files:/overleaf/services/filestore/public_files
- filestore-template-files:/overleaf/services/filestore/template_files
- filestore-uploads:/overleaf/services/filestore/uploads
history-v1:
build:
context: ..
dockerfile: services/history-v1/Dockerfile
env_file:
- dev.env
environment:
OVERLEAF_EDITOR_ANALYTICS_BUCKET: "/buckets/analytics"
OVERLEAF_EDITOR_BLOBS_BUCKET: "/buckets/blobs"
OVERLEAF_EDITOR_CHUNKS_BUCKET: "/buckets/chunks"
OVERLEAF_EDITOR_PROJECT_BLOBS_BUCKET: "/buckets/project_blobs"
OVERLEAF_EDITOR_ZIPS_BUCKET: "/buckets/zips"
PERSISTOR_BACKEND: fs
volumes:
- history-v1-buckets:/buckets
mongo:
image: mongo:8
command: --replSet overleaf
ports:
- "127.0.0.1:27017:27017" # for debugging
volumes:
- mongo-data:/data/db
- ../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
environment:
MONGO_INITDB_DATABASE: sharelatex
extra_hosts:
# Required when using the automatic database setup for initializing the
# replica set. This override is not needed when running the setup after
# starting up mongo.
- mongo:127.0.0.1
notifications:
build:
context: ..
dockerfile: services/notifications/Dockerfile
env_file:
- dev.env
project-history:
build:
context: ..
dockerfile: services/project-history/Dockerfile
env_file:
- dev.env
real-time:
build:
context: ..
dockerfile: services/real-time/Dockerfile
env_file:
- dev.env
redis:
image: redis:7
ports:
- "127.0.0.1:6379:6379" # for debugging
volumes:
- redis-data:/data
web:
build:
context: ..
dockerfile: services/web/Dockerfile
target: dev
env_file:
- dev.env
environment:
- APP_NAME=Overleaf Community Edition
- ENABLED_LINKED_FILE_TYPES=project_file,project_output_file
- EMAIL_CONFIRMATION_DISABLED=true
- NODE_ENV=development
- OVERLEAF_ALLOW_PUBLIC_ACCESS=true
command: ["node", "app.mjs"]
volumes:
- sharelatex-data:/var/lib/overleaf
- web-data:/overleaf/services/web/data
depends_on:
- mongo
- redis
- chat
- clsi
- docstore
- document-updater
- filestore
- history-v1
- notifications
- project-history
- real-time
webpack:
build:
context: ..
dockerfile: services/web/Dockerfile
target: webpack
command:
["npx", "webpack", "serve", "--config", "webpack.config.dev-env.js"]
ports:
- "127.0.0.1:80:3808"
volumes:
- ./webpack.config.dev-env.js:/overleaf/services/web/webpack.config.dev-env.js
-23
View File
@@ -1,23 +0,0 @@
const { merge } = require('webpack-merge')
const base = require('./webpack.config.dev')
module.exports = merge(base, {
devServer: {
allowedHosts: 'auto',
devMiddleware: {
index: false,
},
proxy: [
{
context: '/socket.io/**',
target: 'http://real-time:3026',
ws: true,
},
{
context: ['!**/*.js', '!**/*.css', '!**/*.json'],
target: 'http://web:3000',
},
],
},
})
BIN
View File
Binary file not shown.

Before

Width:  |  Height:  |  Size: 13 KiB

After

Width:  |  Height:  |  Size: 71 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 271 KiB

After

Width:  |  Height:  |  Size: 739 KiB

+18 -21
View File
@@ -1,23 +1,20 @@
version: "2.2"
version: '2.2'
services:
sharelatex:
ports:
- 30000:30000
- 30150:30150
- 30120:30120
- 30050:30050
- 30420:30420
- 30030:30030
- 30160:30160
- 30360:30360
- 30130:30130
- 30100:30100
- 30540:30540
- 30640:30640
- 40000:40000
sharelatex:
ports:
- 40000:40000
- 30150:30150
- 30120:30120
- 30050:30050
- 30420:30420
- 30030:30030
- 30160:30160
- 30360:30360
- 30130:30130
- 30100:30100
# Server Pro
- 30070:30070
- 30400:30400
environment:
DEBUG_NODE: "true"
# Server Pro
- 30070:30070
- 30400:30400
environment:
DEBUG_NODE: 'true'
+124 -126
View File
@@ -1,149 +1,147 @@
version: '2.2'
services:
sharelatex:
restart: always
# Server Pro users:
# image: quay.io/sharelatex/sharelatex-pro
image: sharelatex/sharelatex
container_name: sharelatex
depends_on:
mongo:
condition: service_healthy
redis:
condition: service_started
ports:
- 80:80
stop_grace_period: 60s
volumes:
- ~/sharelatex_data:/var/lib/overleaf
########################################################################
#### Server Pro: Uncomment the following line to mount the docker ####
#### socket, required for Sibling Containers to work ####
########################################################################
# - /var/run/docker.sock:/var/run/docker.sock
environment:
OVERLEAF_APP_NAME: Overleaf Community Edition
sharelatex:
restart: always
# Server Pro users:
# image: quay.io/sharelatex/sharelatex-pro
image: sharelatex/sharelatex
container_name: sharelatex
depends_on:
mongo:
condition: service_healthy
redis:
condition: service_started
ports:
- 80:80
links:
- mongo
- redis
volumes:
- ~/sharelatex_data:/var/lib/sharelatex
########################################################################
#### Server Pro: Uncomment the following line to mount the docker ####
#### socket, required for Sibling Containers to work ####
########################################################################
# - /var/run/docker.sock:/var/run/docker.sock
environment:
OVERLEAF_MONGO_URL: mongodb://mongo/sharelatex
SHARELATEX_APP_NAME: Overleaf Community Edition
# Same property, unfortunately with different names in
# different locations
OVERLEAF_REDIS_HOST: redis
REDIS_HOST: redis
SHARELATEX_MONGO_URL: mongodb://mongo/sharelatex
ENABLED_LINKED_FILE_TYPES: "project_file,project_output_file"
# Same property, unfortunately with different names in
# different locations
SHARELATEX_REDIS_HOST: redis
REDIS_HOST: redis
# Enables Thumbnail generation using ImageMagick
ENABLE_CONVERSIONS: "true"
ENABLED_LINKED_FILE_TYPES: 'project_file,project_output_file'
# Disables email confirmation requirement
EMAIL_CONFIRMATION_DISABLED: "true"
# Enables Thumbnail generation using ImageMagick
ENABLE_CONVERSIONS: 'true'
## Set for SSL via nginx-proxy
#VIRTUAL_HOST: 103.112.212.22
# Disables email confirmation requirement
EMAIL_CONFIRMATION_DISABLED: 'true'
# OVERLEAF_SITE_URL: http://overleaf.example.com
# OVERLEAF_NAV_TITLE: Overleaf Community Edition
# OVERLEAF_HEADER_IMAGE_URL: http://example.com/mylogo.png
# OVERLEAF_ADMIN_EMAIL: support@it.com
# temporary fix for LuaLaTex compiles
# see https://github.com/overleaf/overleaf/issues/695
TEXMFVAR: /var/lib/sharelatex/tmp/texmf-var
# OVERLEAF_LEFT_FOOTER: '[{"text": "Another page I want to link to can be found <a href=\"here\">here</a>"} ]'
# OVERLEAF_RIGHT_FOOTER: '[{"text": "Hello I am on the Right"} ]'
## Set for SSL via nginx-proxy
#VIRTUAL_HOST: 103.112.212.22
# OVERLEAF_EMAIL_FROM_ADDRESS: "hello@example.com"
# SHARELATEX_SITE_URL: http://sharelatex.mydomain.com
# SHARELATEX_NAV_TITLE: Our ShareLaTeX Instance
# SHARELATEX_HEADER_IMAGE_URL: http://somewhere.com/mylogo.png
# SHARELATEX_ADMIN_EMAIL: support@it.com
# OVERLEAF_EMAIL_AWS_SES_ACCESS_KEY_ID:
# OVERLEAF_EMAIL_AWS_SES_SECRET_KEY:
# SHARELATEX_LEFT_FOOTER: '[{"text": "Powered by <a href=\"https://www.sharelatex.com\">ShareLaTeX</a> 2016"},{"text": "Another page I want to link to can be found <a href=\"here\">here</a>"} ]'
# SHARELATEX_RIGHT_FOOTER: '[{"text": "Hello I am on the Right"} ]'
# OVERLEAF_EMAIL_SMTP_HOST: smtp.example.com
# OVERLEAF_EMAIL_SMTP_PORT: 587
# OVERLEAF_EMAIL_SMTP_SECURE: false
# OVERLEAF_EMAIL_SMTP_USER:
# OVERLEAF_EMAIL_SMTP_PASS:
# OVERLEAF_EMAIL_SMTP_TLS_REJECT_UNAUTH: true
# OVERLEAF_EMAIL_SMTP_IGNORE_TLS: false
# OVERLEAF_EMAIL_SMTP_NAME: '127.0.0.1'
# OVERLEAF_EMAIL_SMTP_LOGGER: true
# OVERLEAF_CUSTOM_EMAIL_FOOTER: "This system is run by department x"
# SHARELATEX_EMAIL_FROM_ADDRESS: "team@sharelatex.com"
# ENABLE_CRON_RESOURCE_DELETION: true
# SHARELATEX_EMAIL_AWS_SES_ACCESS_KEY_ID:
# SHARELATEX_EMAIL_AWS_SES_SECRET_KEY:
################
## Server Pro ##
################
# SHARELATEX_EMAIL_SMTP_HOST: smtp.mydomain.com
# SHARELATEX_EMAIL_SMTP_PORT: 587
# SHARELATEX_EMAIL_SMTP_SECURE: false
# SHARELATEX_EMAIL_SMTP_USER:
# SHARELATEX_EMAIL_SMTP_PASS:
# SHARELATEX_EMAIL_SMTP_TLS_REJECT_UNAUTH: true
# SHARELATEX_EMAIL_SMTP_IGNORE_TLS: false
# SHARELATEX_EMAIL_SMTP_NAME: '127.0.0.1'
# SHARELATEX_EMAIL_SMTP_LOGGER: true
# SHARELATEX_CUSTOM_EMAIL_FOOTER: "This system is run by department x"
## The Community Edition is intended for use in environments where all users are trusted and is not appropriate for
## scenarios where isolation of users is required. Sandboxed Compiles are not available in the Community Edition,
## so the following environment variables must be commented out to avoid compile issues.
##
## Sandboxed Compiles: https://docs.overleaf.com/on-premises/configuration/overleaf-toolkit/server-pro-only-configuration/sandboxed-compiles
SANDBOXED_COMPILES: "true"
### Bind-mount source for /var/lib/overleaf/data/compiles inside the container.
SANDBOXED_COMPILES_HOST_DIR_COMPILES: "/home/user/sharelatex_data/data/compiles"
### Bind-mount source for /var/lib/overleaf/data/output inside the container.
SANDBOXED_COMPILES_HOST_DIR_OUTPUT: "/home/user/sharelatex_data/data/output"
### Backwards compatibility (before Server Pro 5.5)
DOCKER_RUNNER: "true"
SANDBOXED_COMPILES_SIBLING_CONTAINERS: "true"
################
## Server Pro ##
################
## Works with test LDAP server shown at bottom of docker compose
# OVERLEAF_LDAP_URL: 'ldap://ldap:389'
# OVERLEAF_LDAP_SEARCH_BASE: 'ou=people,dc=planetexpress,dc=com'
# OVERLEAF_LDAP_SEARCH_FILTER: '(uid={{username}})'
# OVERLEAF_LDAP_BIND_DN: 'cn=admin,dc=planetexpress,dc=com'
# OVERLEAF_LDAP_BIND_CREDENTIALS: 'GoodNewsEveryone'
# OVERLEAF_LDAP_EMAIL_ATT: 'mail'
# OVERLEAF_LDAP_NAME_ATT: 'cn'
# OVERLEAF_LDAP_LAST_NAME_ATT: 'sn'
# OVERLEAF_LDAP_UPDATE_USER_DETAILS_ON_LOGIN: 'true'
# SANDBOXED_COMPILES: 'true'
# OVERLEAF_TEMPLATES_USER_ID: "578773160210479700917ee5"
# OVERLEAF_NEW_PROJECT_TEMPLATE_LINKS: '[ {"name":"All Templates","url":"/templates/all"}]'
# SANDBOXED_COMPILES_SIBLING_CONTAINERS: 'true'
# SANDBOXED_COMPILES_HOST_DIR: '/var/sharelatex_data/data/compiles'
# OVERLEAF_PROXY_LEARN: "true"
# DOCKER_RUNNER: 'false'
mongo:
restart: always
image: mongo:8.0
container_name: mongo
command: "--replSet overleaf"
volumes:
- ~/mongo_data:/data/db
- ./bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
environment:
MONGO_INITDB_DATABASE: sharelatex
extra_hosts:
# Required when using the automatic database setup for initializing the replica set.
# This override is not needed when running the setup after starting up mongo.
- mongo:127.0.0.1
healthcheck:
test: echo 'db.stats().ok' | mongosh localhost:27017/test --quiet
interval: 10s
timeout: 10s
retries: 5
## Works with test LDAP server shown at bottom of docker compose
# SHARELATEX_LDAP_URL: 'ldap://ldap:389'
# SHARELATEX_LDAP_SEARCH_BASE: 'ou=people,dc=planetexpress,dc=com'
# SHARELATEX_LDAP_SEARCH_FILTER: '(uid={{username}})'
# SHARELATEX_LDAP_BIND_DN: 'cn=admin,dc=planetexpress,dc=com'
# SHARELATEX_LDAP_BIND_CREDENTIALS: 'GoodNewsEveryone'
# SHARELATEX_LDAP_EMAIL_ATT: 'mail'
# SHARELATEX_LDAP_NAME_ATT: 'cn'
# SHARELATEX_LDAP_LAST_NAME_ATT: 'sn'
# SHARELATEX_LDAP_UPDATE_USER_DETAILS_ON_LOGIN: 'true'
redis:
restart: always
image: redis:6.2
container_name: redis
volumes:
- ~/redis_data:/data
# SHARELATEX_TEMPLATES_USER_ID: "578773160210479700917ee5"
# SHARELATEX_NEW_PROJECT_TEMPLATE_LINKS: '[ {"name":"All Templates","url":"/templates/all"}]'
# ldap:
# restart: always
# image: rroemhild/test-openldap
# container_name: ldap
# See https://github.com/jwilder/nginx-proxy for documentation on how to configure the nginx-proxy container,
# and https://github.com/overleaf/overleaf/wiki/HTTPS-reverse-proxy-using-Nginx for an example of some recommended
# settings. We recommend using a properly managed nginx instance outside of the Overleaf Server Pro setup,
# but the example here can be used if you'd prefer to run everything with docker-compose
# SHARELATEX_PROXY_LEARN: "true"
# nginx-proxy:
# image: jwilder/nginx-proxy
# container_name: nginx-proxy
# ports:
# - "80:80"
# - "443:443"
# volumes:
# - /var/run/docker.sock:/tmp/docker.sock:ro
# - /home/overleaf/tmp:/etc/nginx/certs
mongo:
restart: always
image: mongo:4.0
container_name: mongo
expose:
- 27017
volumes:
- ~/mongo_data:/data/db
healthcheck:
test: echo 'db.stats().ok' | mongo localhost:27017/test --quiet
interval: 10s
timeout: 10s
retries: 5
redis:
restart: always
image: redis:5
container_name: redis
expose:
- 6379
volumes:
- ~/redis_data:/data
# ldap:
# restart: always
# image: rroemhild/test-openldap
# container_name: ldap
# expose:
# - 389
# See https://github.com/jwilder/nginx-proxy for documentation on how to configure the nginx-proxy container,
# and https://github.com/overleaf/overleaf/wiki/HTTPS-reverse-proxy-using-Nginx for an example of some recommended
# settings. We recommend using a properly managed nginx instance outside of the Overleaf Server Pro setup,
# but the example here can be used if you'd prefer to run everything with docker-compose
# nginx-proxy:
# image: jwilder/nginx-proxy
# container_name: nginx-proxy
# ports:
# #- "80:80"
# - "443:443"
# volumes:
# - /var/run/docker.sock:/tmp/docker.sock:ro
# - /home/sharelatex/tmp:/etc/nginx/certs
-18
View File
@@ -1,18 +0,0 @@
FROM cypress/included:15.12.0
ARG USER_UID=1000
ARG USER_GID=1000
# Corepack setup, shared between all the images.
ENV PATH="/overleaf/node_modules/.bin:$PATH"
ENV COREPACK_HOME=/opt/corepack
RUN corepack enable && corepack install -g yarn@4.14.1
ENV COREPACK_ENABLE_NETWORK=0
WORKDIR /overleaf
RUN sed -i s/node:x:1000:/node:x:${USER_GID}:/ /etc/group \
&& sed -i s_node:x:1000:1000::/home/node:/bin/bash_node:x:${USER_UID}:${USER_GID}::/home/node:/bin/bash_ /etc/passwd \
&& chown -R node:node /home/node \
&& chown node:node /overleaf
USER node
-100
View File
@@ -1,100 +0,0 @@
# Design: per-project Python dependencies (cached virtualenv)
Status: **Phase 1 implemented** (gated behind `OVERLEAF_ENABLE_PROJECT_PYTHON_VENV`,
on in the deployment). Network egress policy and venv eviction (Phases 23)
remain. Captures the plan for letting Quarto `{python}` cells use libraries
beyond the curated base set.
## What ships in Phase 1
- A project root `requirements.vrf` is installed into a venv cached by its
sha256, created with `python3 -m venv --system-site-packages`; `QuartoRunner`
points Quarto at it via `QUARTO_PYTHON`. A per-hash `flock` serialises
concurrent builds; pip output is merged into `output.log`; on failure the
render falls back to the base interpreter (and the missing-package message
surfaces). Venvs live under `PYTHON_VENVS_DIR`
(default `/var/lib/overleaf/data/python-venvs`).
- Gated by `userCanInstallPython` (`PythonVenvGate.mjs`) to the project owner +
invited collaborators (any role) — never anonymous / link-sharing users —
threaded to CLSI as `allowPythonInstall` on the editor compile, presentation
export, and publish paths.
### Known Phase-1 limitations
- The first build of a heavy `requirements.vrf` runs within the compile
timeout; a very large install can be killed and retried next compile (the
venv is only marked complete on success).
- No egress restriction yet (Phase 2) — installs reach PyPI directly.
- No eviction yet (Phase 3) — venvs accumulate under `PYTHON_VENVS_DIR`.
## Background
Quarto executes `` ```{python} `` cells through a Jupyter kernel. The base image
([`server-ce/Dockerfile-base`](../server-ce/Dockerfile-base)) bundles a curated
scientific stack (numpy, pandas, scipy, matplotlib, seaborn, scikit-learn,
sympy, plotly, tabulate). Anything outside that set currently fails the render
with `ModuleNotFoundError`.
As a first step that already shipped, the Quarto log parser
([`quarto-log-parser.ts`](../services/web/frontend/js/ide/log-parser/quarto-log-parser.ts))
turns a missing-package traceback into an actionable message. This document is
the *next* step: letting a project declare and install its own dependencies.
**Key constraint:** the instance runs with anonymous read+write enabled
(`OVERLEAF_ALLOW_ANONYMOUS_READ_AND_WRITE_SHARING=true`), so compiles can be
triggered by untrusted users. Installing arbitrary packages is therefore a
security decision, not just a convenience.
## Mechanism
1. **Declaration.** A standard `requirements.vrf` at the project root opts the
project in (familiar, Quarto-agnostic, supports version pinning).
2. **Keying.** CLSI hashes `sha256(requirements.vrf + python version)`. The hash
names a venv directory on a **persistent volume**, e.g.
`…/data/python-venvs/<hash>/`. Identical dependency sets share one venv across
projects and compiles.
3. **Build-if-missing.** `python3 -m venv --system-site-packages <dir>` (so the
bundled stack stays visible and only the *extra* deps are installed — smaller
and faster), then `<dir>/bin/pip install -r requirements.vrf`. Guard with a
per-hash `flock` so concurrent compiles don't build the same venv twice.
4. **Point Quarto at it.** Set `QUARTO_PYTHON=<dir>/bin/python3` in the render
environment (threaded web → CLSI exactly like `exportMode`). With
`--system-site-packages`, `ipykernel` from the base is importable, so the
kernel runs in that interpreter with base + project packages.
## Guard rails
- **Auth gating.** Only run the install path for **logged-in owner/collaborator**
compiles. Anonymous-link compiles use the plain base interpreter and never
trigger installs. Web decides and passes a boolean to CLSI; default-deny.
- **Network egress.** The compile environment must reach PyPI to install.
Restrict egress to PyPI / an internal mirror only (k8s NetworkPolicy + pip
`--index-url`), not arbitrary hosts.
- **Resource caps.** Install timeout, venv size cap, max package count; surface
overruns as a clear log error.
- **Trust boundary.** Even gated, a trusted user installing packages is
arbitrary code execution in the sandbox. Containment stays the CLSI container
+ resource limits + egress policy. This is owner-trust-level by design.
## Lifecycle
- **Eviction.** `touch` the venv on use; an LRU cleanup job prunes the oldest
venvs when the volume exceeds a size budget.
- **Failure UX.** pip errors flow into the log panel (reusing the friendly-error
pattern) showing pip's output.
## Rollout
- **Phase 1.** Detection + `flock` venv build + `QUARTO_PYTHON`, behind a
settings flag (default **off**), gated to logged-in owner, dev volume.
- **Phase 2.** Egress NetworkPolicy + index pinning + eviction job.
- **Phase 3.** Nicer pip-error surfacing + a small project-settings UI
affordance.
## Open decisions
- `requirements.vrf` vs a frontmatter field vs both?
- Shared global venv volume vs per-user namespacing (sharing is cheaper;
per-user is stricter isolation)?
- Allow native/compiled wheels (broader support) vs wheels-only/no-build
(tighter security)?
@@ -0,0 +1,73 @@
// this file was auto-generated, do not edit it directly.
// instead run bin/update_build_scripts from
// https://github.com/sharelatex/sharelatex-dev-environment
{
"extends": [
"eslint:recommended",
"standard",
"prettier"
],
"parserOptions": {
"ecmaVersion": 2018
},
"plugins": [
"mocha",
"chai-expect",
"chai-friendly"
],
"env": {
"node": true,
"mocha": true
},
"rules": {
// Swap the no-unused-expressions rule with a more chai-friendly one
"no-unused-expressions": 0,
"chai-friendly/no-unused-expressions": "error",
// Do not allow importing of implicit dependencies.
"import/no-extraneous-dependencies": "error"
},
"overrides": [
{
// Test specific rules
"files": ["test/**/*.js"],
"globals": {
"expect": true
},
"rules": {
// mocha-specific rules
"mocha/handle-done-callback": "error",
"mocha/no-exclusive-tests": "error",
"mocha/no-global-tests": "error",
"mocha/no-identical-title": "error",
"mocha/no-nested-tests": "error",
"mocha/no-pending-tests": "error",
"mocha/no-skipped-tests": "error",
"mocha/no-mocha-arrows": "error",
// chai-specific rules
"chai-expect/missing-assertion": "error",
"chai-expect/terminating-properties": "error",
// prefer-arrow-callback applies to all callbacks, not just ones in mocha tests.
// we don't enforce this at the top-level - just in tests to manage `this` scope
// based on mocha's context mechanism
"mocha/prefer-arrow-callback": "error"
}
},
{
// Backend specific rules
"files": ["lib/**/*.js", "index.js"],
"rules": {
// don't allow console.log in backend code
"no-console": "error",
// Do not allow importing of implicit dependencies.
"import/no-extraneous-dependencies": ["error", {
// Do not allow importing of devDependencies.
"devDependencies": false
}]
}
}
]
}
@@ -0,0 +1,47 @@
compileFolder
Compiled source #
###################
*.com
*.class
*.dll
*.exe
*.o
*.so
# Packages #
############
# it's better to unpack these files and commit the raw source
# git has its own built in compression methods
*.7z
*.dmg
*.gz
*.iso
*.jar
*.rar
*.tar
*.zip
# Logs and databases #
######################
*.log
*.sql
*.sqlite
# OS generated files #
######################
.DS_Store?
ehthumbs.db
Icon?
Thumbs.db
/node_modules/*
data/*/*
**.swp
/log.json
hash_folder
.npmrc
Dockerfile
@@ -1,13 +0,0 @@
let reporterOptions = {}
if (process.env.CI) {
reporterOptions = {
reporter: require.resolve('mocha-multi-reporters'),
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
}
}
const all = {
require: 'test/setup.js',
...reporterOptions,
}
module.exports = all
-1
View File
@@ -1 +0,0 @@
24.14.1
@@ -0,0 +1,11 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
{
"arrowParens": "avoid",
"semi": false,
"singleQuote": true,
"trailingComma": "es5",
"tabWidth": 2,
"useTabs": false
}
@@ -1,10 +1,9 @@
access-token-encryptor
--dependencies=None
--docker-repos=gcr.io/overleaf-ops
--env-add=
--env-pass-through=
--esmock-loader=False
--is-library=True
--node-version=24.14.1
--package-name=@overleaf/access-token-encryptor
--pipeline-owner=32
--node-version=12.22.3
--public-repo=False
--script-version=3.11.0
@@ -1,163 +1,116 @@
const { promisify } = require('node:util')
const crypto = require('node:crypto')
const crypto = require('crypto')
const logger = require('logger-sharelatex')
const ALGORITHM = 'aes-256-ctr'
const cryptoHkdf = promisify(crypto.hkdf)
const cryptoRandomBytes = promisify(crypto.randomBytes)
const keyFn = (password, salt, callback) =>
crypto.pbkdf2(password, salt, 10000, 64, 'sha1', callback)
class AbstractAccessTokenScheme {
constructor(cipherLabel, cipherPassword) {
this.cipherLabel = cipherLabel
this.cipherPassword = cipherPassword
}
/**
* @param {Object} json
* @return {Promise<string>}
*/
async encryptJson(json) {
throw new Error('encryptJson is not implemented')
}
/**
* @param {string} encryptedJson
* @return {Promise<Object>}
*/
async decryptToJson(encryptedJson) {
throw new Error('decryptToJson is not implemented')
}
}
class AccessTokenSchemeWithGenericKeyFn extends AbstractAccessTokenScheme {
/**
* @param {Buffer} salt
* @return {Promise<Buffer>}
*/
async keyFn(salt) {
throw new Error('keyFn is not implemented')
}
async encryptJson(json) {
const plainText = JSON.stringify(json)
const bytes = await cryptoRandomBytes(32)
const salt = bytes.slice(0, 16)
const iv = bytes.slice(16, 32)
const key = await this.keyFn(salt)
const cipher = crypto.createCipheriv(ALGORITHM, key, iv)
const cipherText =
cipher.update(plainText, 'utf8', 'base64') + cipher.final('base64')
return [
this.cipherLabel,
salt.toString('hex'),
cipherText,
iv.toString('hex'),
].join(':')
}
async decryptToJson(encryptedJson) {
const [, salt, cipherText, iv] = encryptedJson.split(':', 4)
const key = await this.keyFn(Buffer.from(salt, 'hex'))
const decipher = crypto.createDecipheriv(
ALGORITHM,
key,
Buffer.from(iv, 'hex')
)
const plainText =
decipher.update(cipherText, 'base64', 'utf8') + decipher.final('utf8')
try {
return JSON.parse(plainText)
} catch (e) {
throw new Error('error decrypting token')
}
}
}
class AccessTokenSchemeV3 extends AccessTokenSchemeWithGenericKeyFn {
async keyFn(salt) {
const optionalInfo = ''
return await cryptoHkdf(
'sha512',
this.cipherPassword,
salt,
optionalInfo,
32
)
}
}
const keyFn32 = (password, salt, keyLength, callback) =>
crypto.pbkdf2(password, salt, 10000, 32, 'sha1', callback)
class AccessTokenEncryptor {
constructor(settings) {
/**
* @type {Map<string, AbstractAccessTokenScheme>}
*/
this.schemeByCipherLabel = new Map()
for (const cipherLabel of Object.keys(settings.cipherPasswords)) {
if (!cipherLabel) {
throw new Error('cipherLabel cannot be empty')
}
if (cipherLabel.match(/:/)) {
throw new Error(
`cipherLabel must not contain a colon (:), got ${cipherLabel}`
)
}
const [, version] = cipherLabel.split('-')
if (!version) {
throw new Error(
`cipherLabel must contain version suffix (e.g. 2042.1-v42), got ${cipherLabel}`
)
}
const cipherPassword = settings.cipherPasswords[cipherLabel]
if (!cipherPassword) {
throw new Error(`cipherPasswords['${cipherLabel}'] is missing`)
}
if (cipherPassword.length < 16) {
throw new Error(`cipherPasswords['${cipherLabel}'] is too short`)
}
let scheme
switch (version) {
case 'v3':
scheme = new AccessTokenSchemeV3(cipherLabel, cipherPassword)
break
default:
throw new Error(`unknown version '${version}' for ${cipherLabel}`)
}
this.schemeByCipherLabel.set(cipherLabel, scheme)
this.settings = settings
this.cipherLabel = this.settings.cipherLabel
if (this.cipherLabel && this.cipherLabel.match(/:/)) {
throw Error('cipherLabel must not contain a colon (:)')
}
/** @type {AbstractAccessTokenScheme} */
this.defaultScheme = this.schemeByCipherLabel.get(settings.cipherLabel)
if (!this.defaultScheme) {
throw new Error(`unknown default cipherLabel ${settings.cipherLabel}`)
this.cipherPassword = this.settings.cipherPasswords[this.cipherLabel]
if (!this.cipherPassword) {
throw Error('cipherPassword not set')
}
if (this.cipherPassword.length < 16) {
throw Error('cipherPassword too short')
}
}
promises = {
encryptJson: async json => await this.defaultScheme.encryptJson(json),
decryptToJson: async encryptedJson => {
const [label] = encryptedJson.split(':', 1)
const scheme = this.schemeByCipherLabel.get(label)
if (!scheme) {
throw new Error('unknown access-token-encryptor label ' + label)
}
return await scheme.decryptToJson(encryptedJson)
},
}
encryptJson(json, callback) {
this.promises.encryptJson(json).then(s => callback(null, s), callback)
const string = JSON.stringify(json)
crypto.randomBytes(32, (err, bytes) => {
if (err) {
return callback(err)
}
const salt = bytes.slice(0, 16)
const iv = bytes.slice(16, 32)
keyFn32(this.cipherPassword, salt, 32, (err, key) => {
if (err) {
logger.err({ err }, 'error getting Fn key')
return callback(err)
}
const cipher = crypto.createCipheriv(ALGORITHM, key, iv)
const crypted =
cipher.update(string, 'utf8', 'base64') + cipher.final('base64')
callback(
null,
`${this.cipherLabel}:${salt.toString('hex')}:${crypted}:${iv.toString(
'hex'
)}`
)
})
})
}
decryptToJson(encryptedJson, callback) {
this.promises
.decryptToJson(encryptedJson)
.then(o => callback(null, o), callback)
const [label, salt, cipherText, iv] = encryptedJson.split(':', 4)
const password = this.settings.cipherPasswords[label]
if (!password || password.length < 16) {
return callback(new Error('invalid password'))
}
if (iv) {
this.decryptToJsonV2(password, salt, cipherText, iv, callback)
} else {
this.decryptToJsonV1(password, salt, cipherText, callback)
}
}
decryptToJsonV1(password, salt, cipherText, callback) {
keyFn(password, Buffer.from(salt, 'hex'), (err, key) => {
let json
if (err) {
logger.err({ err }, 'error getting Fn key')
return callback(err)
}
// eslint-disable-next-line node/no-deprecated-api
const decipher = crypto.createDecipher(ALGORITHM, key)
const dec =
decipher.update(cipherText, 'base64', 'utf8') + decipher.final('utf8')
try {
json = JSON.parse(dec)
} catch (e) {
return callback(new Error('error decrypting token'))
}
callback(null, json, true)
})
}
decryptToJsonV2(password, salt, cipherText, iv, callback) {
keyFn32(password, Buffer.from(salt, 'hex'), 32, (err, key) => {
let json
if (err) {
logger.err({ err }, 'error getting Fn key')
return callback(err)
}
const decipher = crypto.createDecipheriv(
ALGORITHM,
key,
Buffer.from(iv, 'hex')
)
const dec =
decipher.update(cipherText, 'base64', 'utf8') + decipher.final('utf8')
try {
json = JSON.parse(dec)
} catch (e) {
return callback(new Error('error decrypting token'))
}
callback(null, json)
})
}
}
File diff suppressed because it is too large Load Diff
+28 -16
View File
@@ -1,28 +1,40 @@
{
"name": "@overleaf/access-token-encryptor",
"version": "3.0.0",
"version": "2.1.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "yarn run lint && yarn run types:check && yarn run test:unit",
"lint": "eslint --cache --cache-location ../../node_modules/.cache/eslint/ --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --cache --cache-location ../../node_modules/.cache/eslint/ --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
"test:ci": "yarn run test:unit",
"test:unit": "mocha --exit test/**/*.{js,cjs}",
"types:check": "tsc --noEmit"
"test": "mocha test/**/*.js",
"lint": "eslint --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix .",
"format": "prettier --list-different $PWD/'**/*.js'",
"format:fix": "prettier --write $PWD/'**/*.js'",
"test:ci": "npm run test"
},
"author": "",
"license": "AGPL-3.0-only",
"dependencies": {
"lodash": "^4.18.1"
"dependencies": {},
"peerDependencies": {
"logger-sharelatex": "^2.2.0"
},
"devDependencies": {
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"mocha": "^11.1.0",
"mocha-junit-reporter": "^2.2.1",
"mocha-multi-reporters": "^1.5.1",
"sandboxed-module": "^2.0.4",
"typescript": "^5.0.4"
"bunyan": "^1.8.15",
"chai": "^4.3.4",
"eslint": "^7.21.0",
"eslint-config-prettier": "^8.1.0",
"eslint-config-standard": "^16.0.2",
"eslint-plugin-chai-expect": "^2.2.0",
"eslint-plugin-chai-friendly": "^0.6.0",
"eslint-plugin-import": "^2.22.1",
"eslint-plugin-mocha": "^8.0.0",
"eslint-plugin-node": "^11.1.0",
"eslint-plugin-prettier": "^3.1.2",
"eslint-plugin-promise": "^4.2.1",
"logger-sharelatex": "^2.2.0",
"mocha": "^6.2.2",
"nock": "0.15.2",
"prettier": "^2.2.1",
"sandboxed-module": "^2.0.3",
"sinon": "^7.5.0"
}
}
@@ -1,27 +0,0 @@
function formatTokenUsageStats(STATS) {
const prettyStats = []
const sortedStats = Object.entries(STATS).sort((a, b) =>
a[0] > b[0] ? 1 : -1
)
const totalByName = {}
for (const [key, n] of sortedStats) {
const [name, version, collectionName, path, label] = key.split(':')
totalByName[name] = (totalByName[name] || 0) + n
prettyStats.push({ name, version, collectionName, path, label, n })
}
for (const row of prettyStats) {
row.percentage = ((100 * row.n) / totalByName[row.name])
.toFixed(2)
.padStart(6)
}
if (prettyStats.length === 0) {
console.warn('---')
console.warn('Found 0 access tokens.')
console.warn('---')
} else {
console.table(prettyStats)
}
}
module.exports = { formatTokenUsageStats }
@@ -1,108 +0,0 @@
const _ = require('lodash')
const { formatTokenUsageStats } = require('./format-usage-stats')
const LOG_EVERY_IN_S = parseInt(process.env.LOG_EVERY_IN_S || '5', 10)
const DRY_RUN = !process.argv.includes('--dry-run=false')
/**
* @param {AccessTokenEncryptor} accessTokenEncryptor
* @param {string} encryptedJson
* @return {Promise<string>}
*/
async function reEncryptTokens(accessTokenEncryptor, encryptedJson) {
return await new Promise((resolve, reject) => {
accessTokenEncryptor.decryptToJson(encryptedJson, (err, json) => {
if (err) return reject(err)
accessTokenEncryptor.encryptJson(json, (err, reEncryptedJson) => {
if (err) return reject(err)
resolve(reEncryptedJson)
})
})
})
}
/**
* @param {AccessTokenEncryptor} accessTokenEncryptor
* @param {Collection} collection
* @param {Object} paths
* @param {Object} queryOptions
* @return {Promise<{}>}
*/
async function reEncryptTokensInCollection({
accessTokenEncryptor,
collection,
paths,
queryOptions,
}) {
const { collectionName } = collection
const stats = {}
let processed = 0
let updatedNUsers = 0
let lastLog = 0
const logProgress = () => {
if (DRY_RUN) {
console.warn(
`processed ${processed} | Would have updated ${updatedNUsers} users`
)
} else {
console.warn(`processed ${processed} | Updated ${updatedNUsers} users`)
}
}
const projection = { _id: 1 }
for (const path of Object.values(paths)) {
projection[path] = 1
}
const cursor = collection.find(
{},
{
...queryOptions,
projection,
}
)
for await (const doc of cursor) {
processed++
let update = null
for (const [name, path] of Object.entries(paths)) {
const blob = _.get(doc, path)
if (!blob) continue
// Schema: LABEL-VERSION:SALT:CIPHERTEXT:IV
const [label] = blob.split(':')
let [, version] = label.split('-')
version = version || 'v2'
const key = [name, version, collectionName, path, label].join(':')
stats[key] = (stats[key] || 0) + 1
if (version === 'v2') {
update = update || {}
update[path] = await reEncryptTokens(accessTokenEncryptor, blob)
}
}
if (Date.now() - lastLog >= LOG_EVERY_IN_S * 1000) {
logProgress()
lastLog = Date.now()
}
if (update) {
updatedNUsers++
const { _id } = doc
if (DRY_RUN) {
console.log('Would upgrade tokens for user', _id, Object.keys(update))
} else {
console.log('Upgrading tokens for user', _id, Object.keys(update))
await collection.updateOne({ _id }, { $set: update })
}
}
}
logProgress()
formatTokenUsageStats(stats)
}
module.exports = {
reEncryptTokensInCollection,
}
@@ -1,9 +0,0 @@
module.exports = {
reporterEnabled: 'spec, mocha-junit-reporter',
mochaJunitReporterReporterOptions: {
mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
includePending: true,
jenkinsMode: true,
output: true,
},
}
@@ -1,13 +0,0 @@
const chai = require('chai')
const chaiAsPromised = require('chai-as-promised')
const SandboxedModule = require('sandboxed-module')
chai.use(chaiAsPromised)
SandboxedModule.configure({
sourceTransformers: {
removeNodePrefix: function (source) {
return source.replace(/require\(['"]node:/g, "require('")
},
},
})
@@ -13,292 +13,110 @@ describe('AccessTokenEncryptor', function () {
'2016.1:76a7d64a444ccee1a515b49c44844a69:m5YSkexUsLjcF4gLncm72+k='
this.encrypted2019 =
'2019.1:627143b2ab185a020c8720253a4c984e:7gnY6Ez3/Y3UWgLHLfBtJsE=:bf75cecb6aeea55b3c060e1122d2a82d'
this.encrypted2023 =
'2023.1-v3:a6dd3781dd6ce93a4134874b505a209c:9TdIDAc8V9SeR0ffSn63Jj4=:d8b2de0b733c81b949993dce229abb4c'
this.badLabel = 'xxxxxx:c7a39310056b694c:jQf+Uh5Den3JREtvc82GW5Q='
this.badKey = '2015.1:d7a39310056b694c:jQf+Uh5Den3JREtvc82GW5Q='
this.badCipherText = '2015.1:c7a39310056b694c:xQf+Uh5Den3JREtvc82GW5Q='
this.settings = {
cipherLabel: '2023.1-v3',
cipherLabel: '2019.1',
cipherPasswords: {
'2023.1-v3': '44444444444444444444444444444444444444',
2016.1: '11111111111111111111111111111111111111',
2015.1: '22222222222222222222222222222222222222',
2019.1: '33333333333333333333333333333333333333',
},
}
this.AccessTokenEncryptor = SandboxedModule.require(modulePath, {
globals: {
Buffer,
},
requires: {
'logger-sharelatex': {
err() {},
},
},
})
this.encryptor = new this.AccessTokenEncryptor(this.settings)
})
describe('invalid settings', function () {
it('should flag missing label', function () {
expect(
() =>
new this.AccessTokenEncryptor({
cipherLabel: '',
cipherPasswords: { '': '' },
})
).to.throw(/cipherLabel cannot be empty/)
describe('encrypt', function () {
it('should encrypt the object', function (done) {
this.encryptor.encryptJson(this.testObject, (err, encrypted) => {
expect(err).to.be.null
encrypted.should.match(
/^2019.1:[0-9a-f]{32}:[a-zA-Z0-9=+/]+:[0-9a-f]{32}$/
)
done()
})
})
it('should flag invalid label with colon', function () {
expect(
() =>
new this.AccessTokenEncryptor({
cipherLabel: '2023:1-v2',
cipherPasswords: { '2023:1-v2': '' },
})
).to.throw(/colon/)
})
it('should flag missing password', function () {
expect(
() =>
new this.AccessTokenEncryptor({
cipherPasswords: { '2023.1-v3': '' },
cipherVersions: { '2023.1-v3': 'v3' },
})
).to.throw(/cipherPasswords.+ missing/)
expect(
() =>
new this.AccessTokenEncryptor({
cipherLabel: '2023.1-v3',
cipherPasswords: { '2023.1-v3': undefined },
})
).to.throw(/cipherPasswords.+ missing/)
})
it('should flag short password', function () {
expect(
() =>
new this.AccessTokenEncryptor({
cipherLabel: '2023.1-v3',
cipherPasswords: { '2023.1-v3': 'foo' },
})
).to.throw(/cipherPasswords.+ too short/)
})
it('should flag missing version', function () {
expect(
() =>
new this.AccessTokenEncryptor({
cipherLabel: '2023.1',
cipherPasswords: { 2023.1: '11111111111111111111111111111111' },
})
).to.throw(/must contain version suffix/)
expect(
() =>
new this.AccessTokenEncryptor({
cipherLabel: '2023.1-',
cipherPasswords: { '2023.1-': '11111111111111111111111111111111' },
})
).to.throw(/must contain version suffix/)
})
it('should flag invalid version', function () {
expect(
() =>
new this.AccessTokenEncryptor({
cipherLabel: '2023.1-v0',
cipherPasswords: {
'2023.1-v0': '11111111111111111111111111111111',
},
})
).to.throw(/unknown version/)
})
it('should flag unknown default scheme', function () {
expect(
() =>
new this.AccessTokenEncryptor({
cipherLabel: '2000.1-v3',
cipherPasswords: {
'2023.1-v3': '11111111111111111111111111111111',
},
})
).to.throw(/unknown default cipherLabel/)
it('should encrypt the object differently the next time', function (done) {
this.encryptor.encryptJson(this.testObject, (err, encrypted1) => {
expect(err).to.be.null
this.encryptor.encryptJson(this.testObject, (err, encrypted2) => {
expect(err).to.be.null
encrypted1.should.not.equal(encrypted2)
done()
})
})
})
})
describe('sync', function () {
describe('encrypt', function () {
it('should encrypt the object', function (done) {
this.encryptor.encryptJson(this.testObject, (err, encrypted) => {
expect(err).to.be.null
encrypted.should.match(
/^2023.1-v3:[0-9a-f]{32}:[a-zA-Z0-9=+/]+:[0-9a-f]{32}$/
)
done()
})
})
it('should encrypt the object differently the next time', function (done) {
this.encryptor.encryptJson(this.testObject, (err, encrypted1) => {
expect(err).to.be.null
this.encryptor.encryptJson(this.testObject, (err, encrypted2) => {
expect(err).to.be.null
encrypted1.should.not.equal(encrypted2)
done()
})
})
})
})
describe('decrypt', function () {
it('should decrypt the string to get the same object', function (done) {
this.encryptor.encryptJson(this.testObject, (err, encrypted) => {
expect(err).to.be.null
this.encryptor.decryptToJson(encrypted, (err, decrypted) => {
expect(err).to.be.null
expect(decrypted).to.deep.equal(this.testObject)
done()
})
})
})
it('should not be able to decrypt 2015 string', function (done) {
this.encryptor.decryptToJson(this.encrypted2015, (err, decrypted) => {
expect(err).to.exist
expect(err.message).to.equal(
'unknown access-token-encryptor label 2015.1'
)
expect(decrypted).to.not.exist
done()
})
})
it('should not be able to decrypt a 2016 string', function (done) {
this.encryptor.decryptToJson(this.encrypted2016, (err, decrypted) => {
expect(err).to.exist
expect(err.message).to.equal(
'unknown access-token-encryptor label 2016.1'
)
expect(decrypted).to.not.exist
done()
})
})
it('should not be able to decrypt a 2019 string', function (done) {
this.encryptor.decryptToJson(this.encrypted2019, (err, decrypted) => {
expect(err).to.exist
expect(err.message).to.equal(
'unknown access-token-encryptor label 2019.1'
)
expect(decrypted).to.not.exist
done()
})
})
it('should decrypt an 2023 string to get the same object', function (done) {
this.encryptor.decryptToJson(this.encrypted2023, (err, decrypted) => {
describe('decrypt', function () {
it('should decrypt the string to get the same object', function (done) {
this.encryptor.encryptJson(this.testObject, (err, encrypted) => {
expect(err).to.be.null
this.encryptor.decryptToJson(encrypted, (err, decrypted) => {
expect(err).to.be.null
expect(decrypted).to.deep.equal(this.testObject)
done()
})
})
it('should return an error when decrypting an invalid label', function (done) {
this.encryptor.decryptToJson(this.badLabel, (err, decrypted) => {
expect(err).to.be.instanceof(Error)
expect(decrypted).to.be.undefined
done()
})
})
it('should return an error when decrypting an invalid key', function (done) {
this.encryptor.decryptToJson(this.badKey, (err, decrypted) => {
expect(err).to.be.instanceof(Error)
expect(decrypted).to.be.undefined
done()
})
})
it('should return an error when decrypting an invalid ciphertext', function (done) {
this.encryptor.decryptToJson(this.badCipherText, (err, decrypted) => {
expect(err).to.be.instanceof(Error)
expect(decrypted).to.be.undefined
done()
})
})
})
})
describe('async', function () {
describe('encrypt', function () {
it('should encrypt the object', async function () {
const encrypted = await this.encryptor.promises.encryptJson(
this.testObject
)
encrypted.should.match(
/^2023.1-v3:[0-9a-f]{32}:[a-zA-Z0-9=+/]+:[0-9a-f]{32}$/
)
})
it('should encrypt the object differently the next time', async function () {
const encrypted1 = await this.encryptor.promises.encryptJson(
this.testObject
)
const encrypted2 = await this.encryptor.promises.encryptJson(
this.testObject
)
encrypted1.should.not.equal(encrypted2)
it('should decrypt an 2015 string to get the same object', function (done) {
this.encryptor.decryptToJson(this.encrypted2015, (err, decrypted) => {
expect(err).to.be.null
expect(decrypted).to.deep.equal(this.testObject)
done()
})
})
describe('decrypt', function () {
it('should decrypt the string to get the same object', async function () {
const encrypted = await this.encryptor.promises.encryptJson(
this.testObject
)
const decrypted = await this.encryptor.promises.decryptToJson(encrypted)
it('should decrypt an 2016 string to get the same object', function (done) {
this.encryptor.decryptToJson(this.encrypted2016, (err, decrypted) => {
expect(err).to.be.null
expect(decrypted).to.deep.equal(this.testObject)
done()
})
})
it('should not be able to decrypt 2015 string', async function () {
await expect(
this.encryptor.promises.decryptToJson(this.encrypted2015)
).to.eventually.be.rejectedWith(
'unknown access-token-encryptor label 2015.1'
)
})
it('should not be able to decrypt a 2016 string', async function () {
await expect(
this.encryptor.promises.decryptToJson(this.encrypted2016)
).to.be.rejectedWith('unknown access-token-encryptor label 2016.1')
})
it('should not be able to decrypt a 2019 string', async function () {
await expect(
this.encryptor.promises.decryptToJson(this.encrypted2019)
).to.be.rejectedWith('unknown access-token-encryptor label 2019.1')
})
it('should decrypt an 2023 string to get the same object', async function () {
const decrypted = await this.encryptor.promises.decryptToJson(
this.encrypted2023
)
it('should decrypt an 2019 string to get the same object', function (done) {
this.encryptor.decryptToJson(this.encrypted2019, (err, decrypted) => {
expect(err).to.be.null
expect(decrypted).to.deep.equal(this.testObject)
done()
})
})
it('should return an error when decrypting an invalid label', async function () {
await expect(
this.encryptor.promises.decryptToJson(this.badLabel)
).to.be.rejectedWith('unknown access-token-encryptor label xxxxxx')
it('should return an error when decrypting an invalid label', function (done) {
this.encryptor.decryptToJson(this.badLabel, (err, decrypted) => {
expect(err).to.be.instanceof(Error)
expect(decrypted).to.be.undefined
done()
})
})
it('should return an error when decrypting an invalid key', async function () {
await expect(
this.encryptor.promises.decryptToJson(this.badKey)
).to.be.rejectedWith('unknown access-token-encryptor label 2015.1')
it('should return an error when decrypting an invalid key', function (done) {
this.encryptor.decryptToJson(this.badKey, (err, decrypted) => {
expect(err).to.be.instanceof(Error)
expect(decrypted).to.be.undefined
done()
})
})
it('should return an error when decrypting an invalid ciphertext', async function () {
await expect(
this.encryptor.promises.decryptToJson(this.badCipherText)
).to.be.rejectedWith('unknown access-token-encryptor label 2015.1')
it('should return an error when decrypting an invalid ciphertext', function (done) {
this.encryptor.decryptToJson(this.badCipherText, (err, decrypted) => {
expect(err).to.be.instanceof(Error)
expect(decrypted).to.be.undefined
done()
})
})
})
@@ -1,4 +0,0 @@
{
"extends": "../../tsconfig.backend.json",
"include": ["**/*.js", "**/*.cjs", "**/*.ts"]
}
-19
View File
@@ -1,19 +0,0 @@
const pkg = require('./package.json')
module.exports = {
meta: {
name: pkg.name,
version: pkg.version,
},
rules: {
'no-unnecessary-trans': require('./no-unnecessary-trans'),
'prefer-kebab-url': require('./prefer-kebab-url'),
'should-unescape-trans': require('./should-unescape-trans'),
'no-generated-editor-themes': require('./no-generated-editor-themes'),
'require-script-runner': require('./require-script-runner'),
'require-vi-doMock-valid-path': require('./require-vi-doMock-valid-path'),
'require-loading-label': require('./require-loading-label'),
'require-cio-snake-case-properties': require('./require-cio-snake-case-properties'),
'no-throw-in-callback': require('./no-throw-in-callback'),
},
}
@@ -1,21 +0,0 @@
module.exports = {
meta: {
type: 'error',
docs: {
description:
'Prohibit CodeMirror themes that are generated in a function',
},
},
create(context) {
return {
':matches(ArrowFunctionExpression, FunctionDeclaration, FunctionExpression) CallExpression > MemberExpression[object.name="EditorView"]:matches([property.name="theme"],[property.name="baseTheme"])'(
node
) {
context.report({
node,
message: `EditorView.theme and EditorView.baseTheme each add CSS to the page for every instance of the theme. Store the theme in a variable and reuse it instead.`,
})
},
}
},
}
@@ -1,52 +0,0 @@
const CALLBACK_PARAM_NAMES = new Set(['cb', 'callback', 'done', 'next'])
function isCallbackParam(param) {
return (
param && param.type === 'Identifier' && CALLBACK_PARAM_NAMES.has(param.name)
)
}
module.exports = {
meta: {
type: 'error',
docs: {
description: 'Disallow throw statements inside callback-based functions',
},
messages: {
noThrowInCallback:
'Pass the error to the callback instead of throwing in callback-based code.',
},
},
create(context) {
// Stack tracks whether each enclosing function is a callback-style function.
// A callback-style function is non-async and has a last param named cb/callback/done/next.
const stack = []
function enterFunction(node) {
const params = node.params
const isCallback =
!node.async &&
params.length > 0 &&
isCallbackParam(params[params.length - 1])
stack.push(isCallback)
}
function exitFunction() {
stack.pop()
}
return {
FunctionDeclaration: enterFunction,
'FunctionDeclaration:exit': exitFunction,
FunctionExpression: enterFunction,
'FunctionExpression:exit': exitFunction,
ArrowFunctionExpression: enterFunction,
'ArrowFunctionExpression:exit': exitFunction,
ThrowStatement(node) {
if (stack[stack.length - 1]) {
context.report({ node, messageId: 'noThrowInCallback' })
}
},
}
},
}
@@ -1,43 +0,0 @@
module.exports = {
meta: {
type: 'problem',
fixable: 'code',
docs: {
description: 'Prohibit Trans with no components or values',
},
},
create(context) {
return {
'JSXOpeningElement[name.name="Trans"]'(node) {
const attributes = new Map(
node.attributes.map(attr => [attr.name.name, attr])
)
if (!attributes.has('components')) {
if (node.parent.children.length > 0) {
context.report({
node,
message: `Trans components must not have child elements`,
})
} else if (attributes.has('values')) {
context.report({
node,
message: `Use t('…') when there are no components`,
})
} else {
context.report({
node,
message: `Use t('…') when there are no components`,
fix(fixer) {
const i18nKey = attributes.get('i18nKey').value.value
// Note: Prettier can fix indentation
return fixer.replaceText(node.parent, `{t('${i18nKey}')}`)
},
})
}
}
},
}
},
}
-19
View File
@@ -1,19 +0,0 @@
{
"name": "@overleaf/eslint-plugin",
"version": "0.1.0",
"author": "Overleaf (https://www.overleaf.com)",
"license": "AGPL-3.0-only",
"main": "index.js",
"dependencies": {
"lodash": "^4.18.1"
},
"devDependencies": {
"@typescript-eslint/parser": "^8.59.4"
},
"peerDependencies": {
"eslint": "^10.4.0"
},
"scripts": {
"test": "node rules.test.js"
}
}
@@ -1,84 +0,0 @@
// URL parts should be kebab-case, but we didn't have this rule in the past.
// The ESLint rule `prefer-kebab-url` will ignore these "legacy" URL parts.
const ignoreWords = {
snake: new Set([
'clear_saml_data',
'confirm_link',
'confirm_university_domain',
'create_recurly_account',
'current_history_content',
'current_user',
'default_email',
'disable_managed_users',
'doc_snapshot',
'enable_history_ranges_support',
'features_override',
'generate_password_reset_url',
'get_assignment',
'get_clone',
'health_check',
'institutional_emails',
'latest_template',
'link_after_saml_response',
'linked_file',
'metrics_segmentation',
'new_users',
'no_autostart_post_gateway',
'personal_info',
'planned_maintenance',
'refresh_features',
'register_admin',
'register_ldap_admin',
'register_saml_admin',
'restore_file',
'revert_file',
'saved_vers',
'send_test_email',
'session_maintenance',
'set_in_session',
'sign_in_to_link',
'split_test',
'sso_configuration_test',
'sso_email',
'sso_enrollment',
'track_changes',
'update_admin',
'user_details',
]),
camel: new Set([
'addWorkflowScope',
'aiErrorAssistant',
'aiFeatureUsage',
'beginAuth',
'brandVariationId',
'closeEditor',
'completeRegistration',
'deactivateOldProjects',
'deletedSubscription',
'disconnectAllUsers',
'editingSession',
'emailSubscription',
'enableManagedUsers',
'externalCollaboration',
'flushProjectToTpds',
'indexAll',
'offboardManagedUser',
'openEditor',
'perfTest',
'pollDropboxForUser',
'resendInvite',
'resendManagedUserInvite',
'salesContactForm',
'showSupport',
]),
other: new Set([
'Project',
'disableSSO',
'enableSSO',
'resendSSOLinkInvite',
'usersCSV',
]),
}
module.exports = { ignoreWords }
@@ -1,91 +0,0 @@
const _ = require('lodash')
const { ignoreWords } = require('./prefer-kebab-url-ignore')
const removeTextBetweenBrackets = text => {
while (text.includes('[') || text.includes('(')) {
text = text.replaceAll(/\[[^[\]]*]/g, '')
text = text.replaceAll(/\([^()]*\)/g, '')
}
return text
}
const shouldIgnoreWord = str =>
str.includes(':') ||
str.includes('(') ||
str === '*' ||
str.match(/^[a-z0-9.]+$/) ||
ignoreWords.snake.has(str) ||
ignoreWords.camel.has(str) ||
ignoreWords.other.has(str)
const getSuggestion = routePath => {
if (typeof routePath === 'string') {
const kebabed = routePath
.split('/')
.map(word => (shouldIgnoreWord(word) ? word : _.kebabCase(word)))
.join('/')
return kebabed === routePath ? null : `'${kebabed}'`
}
if (routePath instanceof RegExp) {
const words = removeTextBetweenBrackets(routePath.source).match(/[\w-]+/g)
if (!words) return routePath
let newSource = routePath.source
for (const word of words) {
if (!shouldIgnoreWord(word)) {
newSource = newSource.replaceAll(
new RegExp(`\\b${word}\\b`, 'g'),
_.kebabCase(word)
)
}
}
const kebabed = new RegExp(newSource, routePath.flags)
return kebabed.source.toString() === routePath.source.toString()
? null
: kebabed
}
}
module.exports = {
meta: {
type: 'problem',
fixable: 'code',
hasSuggestions: true,
docs: {
description: 'Enforce using kebab-case for URL paths',
},
},
create: context => ({
CallExpression(node) {
if (
node.callee.type === 'MemberExpression' &&
node.arguments[0]?.type === 'Literal' &&
[/app/i, /router/i].some(callee =>
typeof callee === 'string'
? node.callee.object.name === callee
: callee.test(node.callee.object.name)
) &&
['get', 'post', 'put', 'delete'].includes(node.callee.property.name)
) {
const routePath = node.arguments[0].value
const suggestion = getSuggestion(routePath)
if (suggestion) {
context.report({
node: node.arguments[0],
message: 'Route path should be in kebab-case.',
suggest: [
{
desc: `Change to kebab-case: ${suggestion}`,
fix: fixer => fixer.replaceText(node.arguments[0], suggestion),
},
],
})
}
}
},
}),
}
@@ -1,111 +0,0 @@
'use strict'
const SNAKE_CASE_RE = /^[a-z][a-z0-9]*(_[a-z0-9]+)*$/
function isSnakeCase(name) {
return SNAKE_CASE_RE.test(name)
}
function getStaticKeyName(property) {
if (property.computed) return null
if (property.key.type === 'Identifier') return property.key.name
if (property.key.type === 'Literal' && typeof property.key.value === 'string')
return property.key.value
return null
}
/**
* Check if a node is a call to CustomerIoHandler.updateUserAttributes()
* and return the attributes argument (2nd argument)
*/
function getUpdateUserAttributesArg(node) {
if (
node.callee.type === 'MemberExpression' &&
node.callee.object.type === 'Identifier' &&
node.callee.object.name === 'CustomerIoHandler' &&
node.callee.property.name === 'updateUserAttributes' &&
node.arguments[1]?.type === 'ObjectExpression'
) {
return node.arguments[1]
}
return null
}
/**
* Check if a node is a call to Modules[.promises].hooks.fire('setUserProperties', ...)
* and return the attributes argument (3rd argument)
*/
function getSetUserPropertiesArg(node) {
const callee = node.callee
if (callee.type !== 'MemberExpression' || callee.property.name !== 'fire') {
return null
}
// Check first argument is 'setUserProperties'
if (
!node.arguments[0] ||
node.arguments[0].type !== 'Literal' ||
node.arguments[0].value !== 'setUserProperties'
) {
return null
}
// Match: Modules.hooks.fire or Modules.promises.hooks.fire
const obj = callee.object
if (obj.type === 'MemberExpression' && obj.property.name === 'hooks') {
const parent = obj.object
// Modules.hooks
if (parent.type === 'Identifier' && parent.name === 'Modules') {
if (node.arguments[2]?.type === 'ObjectExpression') {
return node.arguments[2]
}
}
// Modules.promises.hooks
if (
parent.type === 'MemberExpression' &&
parent.property.name === 'promises' &&
parent.object.type === 'Identifier' &&
parent.object.name === 'Modules'
) {
if (node.arguments[2]?.type === 'ObjectExpression') {
return node.arguments[2]
}
}
}
return null
}
module.exports = {
meta: {
type: 'problem',
docs: {
description:
'Enforce snake_case for Customer.io user property attribute names',
},
},
create(context) {
return {
CallExpression(node) {
const attrsNode =
getUpdateUserAttributesArg(node) || getSetUserPropertiesArg(node)
if (!attrsNode) return
for (const property of attrsNode.properties) {
if (property.type === 'SpreadElement') continue
const keyName = getStaticKeyName(property)
if (keyName === null) continue // skip computed/dynamic keys
if (!isSnakeCase(keyName)) {
context.report({
node: property.key,
message: `Customer.io attribute '{{name}}' must be in snake_case.`,
data: { name: keyName },
})
}
}
},
}
},
}
@@ -1,49 +0,0 @@
module.exports = {
meta: {
type: 'problem',
fixable: null,
docs: {
description:
'Require loadingLabel prop when isLoading is specified on OLButton',
},
schema: [],
},
create(context) {
return {
'JSXOpeningElement[name.name="OLButton"]'(node) {
const attributes = new Map(
node.attributes.map(attr => [attr.name?.name, attr])
)
const isLoadingAttr = attributes.get('isLoading')
const loadingLabelAttr = attributes.get('loadingLabel')
if (isLoadingAttr && !loadingLabelAttr) {
const isLoadingValue = isLoadingAttr.value
if (
!isLoadingValue ||
(isLoadingValue.type === 'JSXExpressionContainer' &&
isLoadingValue.expression.type === 'Literal' &&
isLoadingValue.expression.value === true)
) {
context.report({
node: isLoadingAttr,
message:
'Button with isLoading prop must also specify loadingLabel',
})
} else if (
isLoadingValue.type === 'JSXExpressionContainer' &&
isLoadingValue.expression.type !== 'Literal'
) {
context.report({
node: isLoadingAttr,
message:
'Button with isLoading prop must also specify loadingLabel',
})
}
}
},
}
},
}
@@ -1,28 +0,0 @@
module.exports = {
meta: {
type: 'suggestion',
docs: {
description: 'Require Script Runner for scripts',
},
},
create(context) {
let hasImport = false
return {
ImportDeclaration(node) {
if (node.source.value.endsWith('lib/ScriptRunner.mjs')) {
hasImport = true
}
},
'Program:exit'() {
if (!hasImport) {
context.report({
loc: { line: 1, column: 0 },
message:
'Please use Script Runner for scripts. Refer to the developer manual (https://manual.dev-overleaf.com/development/code/web_scripts/#monitor-script-execution-and-usage-with-script-runner) for more information.',
})
}
},
}
},
}
@@ -1,139 +0,0 @@
const path = require('node:path')
const fs = require('node:fs')
module.exports = {
meta: {
type: 'problem',
docs: {
description: 'Ensure vi.doMock first argument is a resolvable path.',
category: 'Best Practices',
recommended: false,
url: '',
},
fixable: 'code',
hasSuggestions: true,
schema: [],
messages: {
unresolvablePath:
'The path "{{pathValue}}" in vi.doMock() cannot be resolved relative to the current file.',
notAStringLiteral:
'The first argument of vi.doMock() must be (or resolve to) a string literal representing a path.',
noArguments: 'vi.doMock() called with no arguments.',
},
},
create(context) {
const currentFilePath = context.filename
// ESLint can sometimes pass <text> or <input> for snippets not in a file
if (currentFilePath === '<text>' || currentFilePath === '<input>') {
return {}
}
const currentDirectory = path.dirname(currentFilePath)
function canResolve(modulePath) {
try {
require.resolve(path.resolve(currentDirectory, modulePath))
return true
} catch (e) {
const absolutePath = path.resolve(currentDirectory, modulePath)
const extensions = [
'',
'.js',
'.mjs',
'.ts',
'.jsx',
'.tsx',
'.json',
'.node',
'/index.js',
'/index.ts',
] // Add common extensions
for (const ext of extensions) {
if (fs.existsSync(absolutePath + ext)) {
return true
}
}
return false
}
}
return {
CallExpression(node) {
if (
node.callee.type === 'MemberExpression' &&
node.callee.object.type === 'Identifier' &&
node.callee.object.name === 'vi' &&
node.callee.property.type === 'Identifier' &&
node.callee.property.name === 'doMock'
) {
if (node.arguments.length === 0) {
context.report({
node,
messageId: 'noArguments',
})
return
}
const firstArg = node.arguments[0]
let pathValue = firstArg.value
if (
firstArg.type !== 'Literal' ||
typeof firstArg.value !== 'string'
) {
if (firstArg.type === 'Identifier') {
const scope = context.sourceCode.getScope(node)
const variable = scope.variables.find(
v => v.name === firstArg.name
)
if (
variable &&
variable.defs.length > 0 &&
variable.defs[0].node.init &&
variable.defs[0].node.init.type === 'Literal' &&
typeof variable.defs[0].node.init.value === 'string'
) {
pathValue = variable.defs[0].node.init.value
if (canResolve(pathValue)) {
return
}
// If the first argument was a variable that didn't resolve then we can't auto-fix it
}
}
context.report({
node: firstArg,
messageId: 'notAStringLiteral',
})
return
}
if (!pathValue.startsWith('.')) {
return
}
if (!canResolve(pathValue)) {
const mjsPath = pathValue.replace('.js', '.mjs')
const additionalReportOptions = {}
if (canResolve(mjsPath)) {
additionalReportOptions.fix = fixer =>
fixer.replaceText(firstArg, `'${mjsPath}'`)
additionalReportOptions.suggest = [
{
desc: `Replace with "${pathValue.replace('.js', '.mjs')}"`,
fix: fixer => fixer.replaceText(firstArg, `'${mjsPath}'`),
},
]
}
context.report({
node: firstArg,
messageId: 'unresolvablePath',
data: {
pathValue,
},
...additionalReportOptions,
})
}
}
},
}
},
}
-331
View File
@@ -1,331 +0,0 @@
const { RuleTester } = require('eslint')
const tsParser = require('@typescript-eslint/parser')
const noThrowInCallback = require('./no-throw-in-callback')
const preferKebabUrl = require('./prefer-kebab-url')
const noUnnecessaryTrans = require('./no-unnecessary-trans')
const shouldUnescapeTrans = require('./should-unescape-trans')
const noGeneratedEditorThemes = require('./no-generated-editor-themes')
const viDoMockValidPath = require('./require-vi-doMock-valid-path')
const requireCioSnakeCaseProperties = require('./require-cio-snake-case-properties')
const ruleTester = new RuleTester({
languageOptions: {
parser: tsParser,
ecmaVersion: 'latest',
parserOptions: { ecmaFeatures: { jsx: true } },
},
})
ruleTester.run('prefer-kebab-url', preferKebabUrl, {
valid: [
{ code: `app.get('/foo-bar')` },
{ code: `app.get('/foo-bar/:id')` },
{ code: `router.post('/foo-bar')` },
{ code: `router.get('/foo-bar/:id/:name/:age')` },
{ code: `webRouter.get('/foo-bar/:user_id/(ProjectName)/get-info')` },
{ code: `webApp.post('/foo-bar/:user_id/(ProjectName)/get-info')` },
{
code: `router.get(/^\\/download\\/project\\/([^/]*)\\/output\\/output\\.pdf$/)`,
},
{
code: `webRouter.get(/^\\/project\\/([^/]*)\\/user\\/([0-9a-f]+)\\/build\\/([0-9a-f-]+)\\/output\\/(.*)$/)`,
},
],
invalid: [
{
code: `app.get('/fooBar')`,
errors: [
{ message: 'Route path should be in kebab-case.', suggestions: 1 },
],
},
{
code: `app.get('/fooBar/:id')`,
errors: [
{ message: 'Route path should be in kebab-case.', suggestions: 1 },
],
},
{
code: `webRouter.get('/foo_bar/:id/FooBar/:name/fooBar')`,
errors: [
{ message: 'Route path should be in kebab-case.', suggestions: 1 },
],
},
{
code: `router.get(/^\\/downLoad\\/pro-ject\\/([^/]*)\\/OutPut\\/out-put\\.pdf$/)`,
errors: [
{ message: 'Route path should be in kebab-case.', suggestions: 1 },
],
},
],
})
ruleTester.run('no-unnecessary-trans', noUnnecessaryTrans, {
valid: [
{ code: `<Trans i18nKey="test" components={{ strong: <strong/> }}/>` },
],
invalid: [
{
code: `<Trans i18nKey="test" values={{ test: 'foo '}}/>`,
errors: [{ message: `Use t('…') when there are no components` }],
},
{
code: `<Trans i18nKey="test" />`,
errors: [{ message: `Use t('…') when there are no components` }],
output: `{t('test')}`,
},
],
})
ruleTester.run('should-unescape-trans', shouldUnescapeTrans, {
valid: [
{
code: `<Trans i18nKey="test" components={{ strong: <strong/> }}/>`,
},
{
code: `<Trans i18nKey="test" values={{ foo: 'bar' }} components={{ strong: <strong/> }} shouldUnescape tOptions={{ interpolation: { escapeValue: true } }}/>`,
},
],
invalid: [
{
code: `<Trans i18nKey="test" values={{ foo: 'bar' }} components={{ strong: <strong/> }} />`,
errors: [{ message: 'Trans with values must have shouldUnescape' }],
output: `<Trans i18nKey="test" values={{ foo: 'bar' }}\nshouldUnescape components={{ strong: <strong/> }} />`,
},
{
code: `<Trans i18nKey="test" values={{ foo: 'bar' }} components={{ strong: <strong/> }} shouldUnescape />`,
errors: [
{
message:
'Trans with shouldUnescape must have tOptions.interpolation.escapeValue',
},
],
output: `<Trans i18nKey="test" values={{ foo: 'bar' }} components={{ strong: <strong/> }} shouldUnescape\ntOptions={{ interpolation: { escapeValue: true } }} />`,
},
],
})
const noGeneratedEditorThemesError =
'EditorView.theme and EditorView.baseTheme each add CSS to the page for every instance of the theme. Store the theme in a variable and reuse it instead.'
ruleTester.run('no-generated-editor-themes', noGeneratedEditorThemes, {
valid: [
{
code: `EditorView.theme({ '.cm-editor': { color: 'black' } })`,
},
{
code: `const theme = EditorView.theme({ '.cm-editor': { color: 'black' } })`,
},
],
invalid: [
{
code: `function createTheme() { return EditorView.theme({ '.cm-editor': { color: 'black' } }) }`,
errors: [
{
message: noGeneratedEditorThemesError,
},
],
},
{
code: `() => EditorView.theme({ '.cm-editor': { color: 'black' } })`,
errors: [
{
message: noGeneratedEditorThemesError,
},
],
},
{
code: `class Foo { createTheme() { return EditorView.theme({ '.cm-editor': { color: 'black' } }) } }`,
errors: [
{
message: noGeneratedEditorThemesError,
},
],
},
],
})
ruleTester.run('domock-require-valid-path', viDoMockValidPath, {
valid: [
{
code: 'vi.doMock("./require-vi-doMock-valid-path.js")',
filename: __filename,
},
{
code: 'const filename = "./require-vi-doMock-valid-path.js"; vi.doMock(filename);',
filename: __filename,
},
],
invalid: [
{
code: "vi.doMock('./require-vi-doMock-valid-path2')",
filename: __filename,
errors: [
{
message:
'The path "./require-vi-doMock-valid-path2" in vi.doMock() cannot be resolved relative to the current file.',
suggestions: [],
},
],
},
{
code: 'const filename = "./require-vi-doMock-valid-path2.js"; vi.doMock(filename);',
filename: __filename,
errors: [
{
message:
'The first argument of vi.doMock() must be (or resolve to) a string literal representing a path.',
suggestions: [],
},
],
},
],
})
ruleTester.run(
'require-cio-snake-case-properties',
requireCioSnakeCaseProperties,
{
valid: [
// updateUserAttributes with snake_case keys
{
code: `CustomerIoHandler.updateUserAttributes(userId, { plan_type: 'free', group_size: 10 })`,
},
// Modules.promises.hooks.fire with snake_case keys
{
code: `Modules.promises.hooks.fire('setUserProperties', userId, { plan_type: 'free', last_active: 123 })`,
},
// Modules.hooks.fire with snake_case keys
{
code: `Modules.hooks.fire('setUserProperties', userId, { plan_type: 'free' })`,
},
// Single-word keys are valid snake_case
{
code: `CustomerIoHandler.updateUserAttributes(userId, { email: 'a@b.com', role: 'admin' })`,
},
// Computed/dynamic keys are skipped
{
code: `CustomerIoHandler.updateUserAttributes(userId, { [dynamicKey]: true })`,
},
// Spread elements are skipped
{
code: `CustomerIoHandler.updateUserAttributes(userId, { ...existingAttrs })`,
},
// Unrelated function calls are not checked
{
code: `SomeOtherHandler.updateUserAttributes(userId, { camelCase: true })`,
},
// fire() with a different event name is not checked
{
code: `Modules.promises.hooks.fire('someOtherEvent', userId, { camelCase: true })`,
},
],
invalid: [
// camelCase key in updateUserAttributes
{
code: `CustomerIoHandler.updateUserAttributes(userId, { planType: 'free' })`,
errors: [
{
message: `Customer.io attribute 'planType' must be in snake_case.`,
},
],
},
// kebab-case string key
{
code: `CustomerIoHandler.updateUserAttributes(userId, { 'plan-type': 'free' })`,
errors: [
{
message: `Customer.io attribute 'plan-type' must be in snake_case.`,
},
],
},
// PascalCase key
{
code: `CustomerIoHandler.updateUserAttributes(userId, { PlanType: 'free' })`,
errors: [
{
message: `Customer.io attribute 'PlanType' must be in snake_case.`,
},
],
},
// camelCase in Modules.promises.hooks.fire
{
code: `Modules.promises.hooks.fire('setUserProperties', userId, { planType: 'free' })`,
errors: [
{
message: `Customer.io attribute 'planType' must be in snake_case.`,
},
],
},
// camelCase in Modules.hooks.fire
{
code: `Modules.hooks.fire('setUserProperties', userId, { planType: 'free' })`,
errors: [
{
message: `Customer.io attribute 'planType' must be in snake_case.`,
},
],
},
// Multiple invalid keys report multiple errors
{
code: `CustomerIoHandler.updateUserAttributes(userId, { planType: 'free', groupSize: 10, plan_term: 'annual' })`,
errors: [
{
message: `Customer.io attribute 'planType' must be in snake_case.`,
},
{
message: `Customer.io attribute 'groupSize' must be in snake_case.`,
},
],
},
],
}
)
const noThrowInCallbackMessage =
'Pass the error to the callback instead of throwing in callback-based code.'
ruleTester.run('no-throw-in-callback', noThrowInCallback, {
valid: [
// Calling the callback with an error is fine
{ code: `function foo(cb) { cb(new Error()) }` },
// async functions may throw (they return a rejected promise)
{ code: `async function foo(cb) { throw new Error() }` },
// Last param not a callback name — not a callback-style function
{ code: `function foo(data) { throw new Error() }` },
// No params at all
{ code: `function foo() { throw new Error() }` },
// throw inside a nested non-callback function is fine
{ code: `function foo(cb) { [1].map(function() { throw new Error() }) }` },
// throw inside a nested async arrow is fine
{ code: `function foo(cb) { [1].map(async () => { throw new Error() }) }` },
],
invalid: [
{
code: `function foo(cb) { throw new Error() }`,
errors: [{ message: noThrowInCallbackMessage }],
},
{
code: `function foo(callback) { throw new Error() }`,
errors: [{ message: noThrowInCallbackMessage }],
},
{
code: `function foo(done) { throw new Error() }`,
errors: [{ message: noThrowInCallbackMessage }],
},
{
code: `function foo(next) { throw new Error() }`,
errors: [{ message: noThrowInCallbackMessage }],
},
{
code: `function foo(data, cb) { throw new Error() }`,
errors: [{ message: noThrowInCallbackMessage }],
},
{
code: `const foo = (cb) => { throw new Error() }`,
errors: [{ message: noThrowInCallbackMessage }],
},
// throw in a nested callback-style function inside another callback function
{
code: `function foo(cb) { bar(function(done) { throw new Error() }) }`,
errors: [{ message: noThrowInCallbackMessage }],
},
],
})
@@ -1,60 +0,0 @@
module.exports = {
meta: {
type: 'problem',
fixable: 'code',
docs: {
description: 'Ensure that Trans with values has shouldUnescape',
},
},
create(context) {
return {
'JSXOpeningElement[name.name="Trans"]'(node) {
const attributes = new Map(
node.attributes.map(attr => [attr.name.name, attr])
)
if (attributes.has('values') && !attributes.has('shouldUnescape')) {
context.report({
node,
message: 'Trans with values must have shouldUnescape',
fix(fixer) {
return fixer.insertTextAfter(
attributes.get('values'),
'\nshouldUnescape' // Note: Prettier can fix indentation
)
},
})
}
if (attributes.has('values') && attributes.has('shouldUnescape')) {
const tOptions = attributes.get('tOptions')
if (!tOptions) {
context.report({
node,
message:
'Trans with shouldUnescape must have tOptions.interpolation.escapeValue',
fix(fixer) {
return fixer.insertTextAfter(
attributes.get('shouldUnescape'),
'\ntOptions={{ interpolation: { escapeValue: true } }}' // Note: Prettier can fix indentation
)
},
})
} else {
const property = tOptions.value.expression.properties
.find(p => p.key.name === 'interpolation')
?.value.properties.find(p => p.key.name === 'escapeValue')
if (property?.value.value !== true) {
context.report({
node,
message:
'Trans with shouldUnescape must have tOptions.interpolation.escapeValue set to true',
})
}
}
}
},
}
},
}
-4
View File
@@ -1,4 +0,0 @@
{
"extends": "../../tsconfig.backend.json",
"include": ["**/*.js"]
}
-13
View File
@@ -1,13 +0,0 @@
let reporterOptions = {}
if (process.env.CI) {
reporterOptions = {
reporter: require.resolve('mocha-multi-reporters'),
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
}
}
const all = {
require: 'test/setup.js',
...reporterOptions,
}
module.exports = all
-1
View File
@@ -1 +0,0 @@
24.14.1
-10
View File
@@ -1,10 +0,0 @@
fetch-utils
--dependencies=None
--env-add=
--env-pass-through=
--esmock-loader=False
--is-library=True
--node-version=24.14.1
--package-name=@overleaf/fetch-utils
--pipeline-owner=32
--public-repo=False
-354
View File
@@ -1,354 +0,0 @@
const _ = require('lodash')
const { Readable } = require('node:stream')
const OError = require('@overleaf/o-error')
const fetch = require('node-fetch')
const http = require('node:http')
const https = require('node:https')
let logger
function setLogger(loggerInstance) {
logger = loggerInstance
}
/**
* @import { Response } from 'node-fetch'
*/
/**
* Make a request and return the parsed JSON response.
*
* @param {string | URL} url - request URL
* @param {any} [opts] - fetch options
* @return {Promise<any>} the parsed JSON response
* @throws {RequestFailedError} if the response has a failure status code
*/
async function fetchJson(url, opts = {}) {
const { json } = await fetchJsonWithResponse(url, opts)
return json
}
async function fetchJsonWithResponse(url, opts = {}) {
const { fetchOpts, detachSignal } = parseOpts(opts, url)
fetchOpts.headers = fetchOpts.headers ?? {}
fetchOpts.headers.Accept = fetchOpts.headers.Accept ?? 'application/json'
const response = await performRequest(url, fetchOpts, detachSignal)
if (!response.ok) {
const body = await maybeGetResponseBody(response)
throw new RequestFailedError(url, opts, response, body)
}
const json = await response.json()
return { json, response }
}
/**
* Make a request and return a stream.
*
* If the response body is destroyed, the request is aborted.
*
* @param {string | URL} url - request URL
* @param {any} [opts] - fetch options
* @return {Promise<Readable>}
* @throws {RequestFailedError} if the response has a failure status code
*/
async function fetchStream(url, opts = {}) {
const { stream } = await fetchStreamWithResponse(url, opts)
return stream
}
async function fetchStreamWithResponse(url, opts = {}) {
const { fetchOpts, abortController, detachSignal } = parseOpts(opts, url)
const response = await performRequest(url, fetchOpts, detachSignal)
if (!response.ok) {
const body = await maybeGetResponseBody(response)
throw new RequestFailedError(url, opts, response, body)
}
abortOnDestroyedResponse(abortController, response)
const stream = response.body
return { stream, response }
}
/**
* Make a request and discard the response.
*
* @param {string | URL} url - request URL
* @param {any} [opts] - fetch options
* @return {Promise<Response>}
* @throws {RequestFailedError} if the response has a failure status code
*/
async function fetchNothing(url, opts = {}) {
const { fetchOpts, detachSignal } = parseOpts(opts, url)
const response = await performRequest(url, fetchOpts, detachSignal)
if (!response.ok) {
const body = await maybeGetResponseBody(response)
throw new RequestFailedError(url, opts, response, body)
}
await discardResponseBody(response)
return response
}
/**
* Make a request and extract the redirect from the response.
*
* @param {string | URL} url - request URL
* @param {any} [opts] - fetch options
* @return {Promise<string>}
* @throws {RequestFailedError} if the response has a non redirect status code or missing Location header
*/
async function fetchRedirect(url, opts = {}) {
const { location } = await fetchRedirectWithResponse(url, opts)
return location
}
/**
* Make a request and extract the redirect from the response.
*
* @param {string | URL} url - request URL
* @param {object} opts - fetch options
* @return {Promise<{location: string, response: Response}>}
* @throws {RequestFailedError} if the response has a non redirect status code or missing Location header
*/
async function fetchRedirectWithResponse(url, opts = {}) {
const { fetchOpts, detachSignal } = parseOpts(opts, url)
fetchOpts.redirect = 'manual'
const response = await performRequest(url, fetchOpts, detachSignal)
if (response.status < 300 || response.status >= 400) {
const body = await maybeGetResponseBody(response)
throw new RequestFailedError(url, opts, response, body)
}
const location = response.headers.get('Location')
if (!location) {
const body = await maybeGetResponseBody(response)
throw new RequestFailedError(url, opts, response, body).withCause(
new OError('missing Location response header on 3xx response', {
headers: Object.fromEntries(response.headers.entries()),
})
)
}
await discardResponseBody(response)
return { location, response }
}
/**
* Make a request and return a string.
*
* @param {string | URL} url - request URL
* @param {any} [opts] - fetch options
* @return {Promise<string>}
* @throws {RequestFailedError} if the response has a failure status code
*/
async function fetchString(url, opts = {}) {
const { body } = await fetchStringWithResponse(url, opts)
return body
}
async function fetchStringWithResponse(url, opts = {}) {
const { fetchOpts, detachSignal } = parseOpts(opts, url)
const response = await performRequest(url, fetchOpts, detachSignal)
if (!response.ok) {
const body = await maybeGetResponseBody(response)
throw new RequestFailedError(url, opts, response, body)
}
const body = await response.text()
return { body, response }
}
class RequestFailedError extends OError {
constructor(url, opts, response, body) {
super('request failed', {
url,
method: opts.method ?? 'GET',
status: response.status,
})
this.response = response
if (body != null) {
this.body = body
}
}
}
function parseOpts(opts, url) {
const fetchOpts = _.omit(opts, ['json', 'signal', 'basicAuth'])
if (opts.json) {
setupJsonBody(fetchOpts, opts.json)
}
if (opts.basicAuth) {
setupBasicAuth(fetchOpts, opts.basicAuth)
}
const abortController = new AbortController()
fetchOpts.signal = abortController.signal
let detachSignal
if (opts.signal) {
detachSignal = abortOnSignal(abortController, opts.signal)
} else {
let overTimeoutStart
const stack = new Error().stack
const timeout = setTimeout(() => {
overTimeoutStart = process.hrtime.bigint()
}, 120000)
detachSignal = () => {
clearTimeout(timeout)
if (overTimeoutStart && logger) {
logger.warn(
{
url,
method: opts.method ?? 'GET',
overTimeoutMs:
Number(process.hrtime.bigint() - overTimeoutStart) / 1e6,
stack,
},
'Fetch request did not complete within 120 seconds'
)
}
}
}
if (opts.body instanceof Readable) {
abortOnDestroyedRequest(abortController, fetchOpts.body)
}
return { fetchOpts, abortController, detachSignal }
}
function setupJsonBody(fetchOpts, json) {
fetchOpts.body = JSON.stringify(json)
fetchOpts.headers = fetchOpts.headers ?? {}
fetchOpts.headers['Content-Type'] = 'application/json'
}
function setupBasicAuth(fetchOpts, basicAuth) {
fetchOpts.headers = fetchOpts.headers ?? {}
fetchOpts.headers.Authorization =
'Basic ' +
Buffer.from(`${basicAuth.user}:${basicAuth.password}`).toString('base64')
}
function abortOnSignal(abortController, signal) {
const listener = () => {
abortController.abort(signal.reason)
}
if (signal.aborted) {
abortController.abort(signal.reason)
}
signal.addEventListener('abort', listener)
return () => {
signal.removeEventListener('abort', listener)
}
}
function abortOnDestroyedRequest(abortController, stream) {
stream.on('close', () => {
if (!stream.readableEnded) {
abortController.abort()
}
})
}
function abortOnDestroyedResponse(abortController, response) {
response.body.on('close', () => {
if (!response.bodyUsed) {
abortController.abort()
}
})
}
async function performRequest(url, fetchOpts, detachSignal) {
let response
try {
response = await fetch(url, fetchOpts)
} catch (err) {
detachSignal()
if (fetchOpts.body instanceof Readable) {
fetchOpts.body.destroy()
}
throw OError.tag(err, err.message, {
url,
method: fetchOpts.method ?? 'GET',
})
}
response.body.on('close', detachSignal)
if (fetchOpts.body instanceof Readable) {
response.body.on('close', () => {
if (!fetchOpts.body.readableEnded) {
fetchOpts.body.destroy()
}
})
}
return response
}
async function discardResponseBody(response) {
// eslint-disable-next-line no-unused-vars
for await (const chunk of response.body) {
// discard the body
}
}
/**
* @param {Response} response
*/
async function maybeGetResponseBody(response) {
try {
return await response.text()
} catch (err) {
return null
}
}
// Define custom http and https agents with support for connect timeouts
class ConnectTimeoutError extends OError {
constructor(options) {
super('connect timeout', options)
}
}
function withTimeout(createConnection, options, callback) {
if (options.connectTimeout) {
// Wrap createConnection in a timeout
const timer = setTimeout(() => {
socket.destroy(new ConnectTimeoutError(options))
}, options.connectTimeout)
const socket = createConnection(options, (err, stream) => {
clearTimeout(timer)
callback(err, stream)
})
return socket
} else {
// Fallback to default createConnection
return createConnection(options, callback)
}
}
class CustomHttpAgent extends http.Agent {
createConnection(options, callback) {
return withTimeout(super.createConnection.bind(this), options, callback)
}
}
class CustomHttpsAgent extends https.Agent {
createConnection(options, callback) {
return withTimeout(super.createConnection.bind(this), options, callback)
}
}
module.exports = {
fetchJson,
fetchJsonWithResponse,
fetchStream,
fetchStreamWithResponse,
fetchNothing,
fetchRedirect,
fetchRedirectWithResponse,
fetchString,
fetchStringWithResponse,
RequestFailedError,
ConnectTimeoutError,
CustomHttpAgent,
CustomHttpsAgent,
setLogger,
}
-33
View File
@@ -1,33 +0,0 @@
{
"name": "@overleaf/fetch-utils",
"version": "0.1.0",
"description": "utilities for node-fetch",
"main": "index.js",
"scripts": {
"test": "yarn run lint && yarn run types:check && yarn run test:unit",
"lint": "eslint --cache --cache-location ../../node_modules/.cache/eslint/ --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
"lint:fix": "eslint --cache --cache-location ../../node_modules/.cache/eslint/ --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
"test:ci": "yarn run test:unit",
"test:unit": "mocha --exit test/**/*.{js,cjs}",
"types:check": "tsc --noEmit"
},
"author": "Overleaf (https://www.overleaf.com)",
"license": "AGPL-3.0-only",
"devDependencies": {
"@types/node-fetch": "^2.6.13",
"body-parser": "1.20.4",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"express": "4.22.1",
"mocha": "^11.1.0",
"mocha-junit-reporter": "^2.2.1",
"mocha-multi-reporters": "^1.5.1",
"selfsigned": "^5.5.0",
"typescript": "^5.0.4"
},
"dependencies": {
"@overleaf/o-error": "workspace:*",
"lodash": "^4.18.1",
"node-fetch": "^2.7.0"
}
}
@@ -1,9 +0,0 @@
module.exports = {
reporterEnabled: 'spec, mocha-junit-reporter',
mochaJunitReporterReporterOptions: {
mochaFile: `reports/junit-mocha-${process.env.MOCHA_GREP}.xml`,
includePending: true,
jenkinsMode: true,
output: true,
},
}
-4
View File
@@ -1,4 +0,0 @@
const chai = require('chai')
const chaiAsPromised = require('chai-as-promised')
chai.use(chaiAsPromised)
@@ -1,450 +0,0 @@
const { expect } = require('chai')
const fs = require('node:fs')
const events = require('node:events')
const { FetchError, AbortError } = require('node-fetch')
const { Readable } = require('node:stream')
const { pipeline } = require('node:stream/promises')
const { once } = require('node:events')
const { TestServer } = require('./helpers/TestServer')
const selfsigned = require('selfsigned')
const {
fetchJson,
fetchStream,
fetchNothing,
fetchRedirect,
fetchString,
RequestFailedError,
CustomHttpAgent,
CustomHttpsAgent,
} = require('../..')
const HTTP_PORT = 30001
const HTTPS_PORT = 30002
const dns = require('node:dns')
const _originalLookup = dns.lookup
// Custom DNS resolver function
dns.lookup = (hostname, options, callback) => {
if (hostname === 'example.com') {
// If the hostname is our test case, return the ip address for the test server
if (options?.all) {
callback(null, [{ address: '127.0.0.1', family: 4 }])
} else {
callback(null, '127.0.0.1', 4)
}
} else {
// Otherwise, use the default lookup
_originalLookup(hostname, options, callback)
}
}
describe('fetch-utils', function () {
let PUBLIC_CERT
before(async function () {
this.server = new TestServer()
const attrs = [{ name: 'commonName', value: 'example.com' }]
const pems = await selfsigned.generate(attrs, { days: 365, keySize: 2048 })
const PRIVATE_KEY = pems.private
PUBLIC_CERT = pems.cert
await this.server.start(HTTP_PORT, HTTPS_PORT, {
key: PRIVATE_KEY,
cert: PUBLIC_CERT,
})
this.url = path => `http://example.com:${HTTP_PORT}${path}`
this.httpsUrl = path => `https://example.com:${HTTPS_PORT}${path}`
})
beforeEach(function () {
this.server.lastReq = undefined
})
after(async function () {
await this.server.stop()
})
describe('fetchJson', function () {
it('parses a JSON response', async function () {
const json = await fetchJson(this.url('/json/hello'))
expect(json).to.deep.equal({ msg: 'hello' })
})
it('parses JSON in the request', async function () {
const json = await fetchJson(this.url('/json/add'), {
method: 'POST',
json: { a: 2, b: 3 },
})
expect(json).to.deep.equal({ sum: 5 })
})
it('accepts stringified JSON as body', async function () {
const json = await fetchJson(this.url('/json/add'), {
method: 'POST',
body: JSON.stringify({ a: 2, b: 3 }),
headers: { 'Content-Type': 'application/json' },
})
expect(json).to.deep.equal({ sum: 5 })
})
it('throws a FetchError when the payload is not JSON', async function () {
await expect(fetchJson(this.url('/hello'))).to.be.rejectedWith(FetchError)
})
it('aborts the request if JSON parsing fails', async function () {
await expect(fetchJson(this.url('/large'))).to.be.rejectedWith(FetchError)
await expectRequestAborted(this.server.lastReq)
})
it('handles errors when the payload is JSON', async function () {
await expect(fetchJson(this.url('/json/500'))).to.be.rejectedWith(
RequestFailedError
)
await expectRequestAborted(this.server.lastReq)
})
it('handles errors when the payload is not JSON', async function () {
await expect(fetchJson(this.url('/500'))).to.be.rejectedWith(
RequestFailedError
)
await expectRequestAborted(this.server.lastReq)
})
it('supports abort signals', async function () {
await expect(
abortOnceReceived(
signal => fetchJson(this.url('/hang'), { signal }),
this.server
)
).to.be.rejectedWith(AbortError)
await expectRequestAborted(this.server.lastReq)
})
it('supports basic auth', async function () {
const json = await fetchJson(this.url('/json/basic-auth'), {
basicAuth: { user: 'user', password: 'pass' },
})
expect(json).to.deep.equal({ key: 'verysecret' })
})
it("destroys the request body if it doesn't get consumed", async function () {
const stream = Readable.from(infiniteIterator())
await fetchJson(this.url('/json/ignore-request'), {
method: 'POST',
body: stream,
})
expect(stream.destroyed).to.be.true
})
})
describe('fetchStream', function () {
it('returns a stream', async function () {
const stream = await fetchStream(this.url('/large'))
const text = await streamToString(stream)
expect(text).to.equal(this.server.largePayload)
})
it('aborts the request when the stream is destroyed', async function () {
const stream = await fetchStream(this.url('/large'))
stream.destroy()
await expectRequestAborted(this.server.lastReq)
})
it('aborts the request when the request body is destroyed before transfer', async function () {
const stream = Readable.from(infiniteIterator())
const promise = fetchStream(this.url('/hang'), {
method: 'POST',
body: stream,
})
stream.destroy()
await expect(promise).to.be.rejectedWith(AbortError)
await wait(80)
expect(this.server.lastReq).to.be.undefined
})
it('aborts the request when the request body is destroyed during transfer', async function () {
const stream = Readable.from(infiniteIterator())
// Note: this test won't work on `/hang`
const promise = fetchStream(this.url('/sink'), {
method: 'POST',
body: stream,
})
await once(this.server.events, 'request-received')
stream.destroy()
await expect(promise).to.be.rejectedWith(AbortError)
await expectRequestAborted(this.server.lastReq)
})
it('handles errors', async function () {
await expect(fetchStream(this.url('/500'))).to.be.rejectedWith(
RequestFailedError
)
await expectRequestAborted(this.server.lastReq)
})
it('supports abort signals', async function () {
await expect(
abortOnceReceived(
signal => fetchStream(this.url('/hang'), { signal }),
this.server
)
).to.be.rejectedWith(AbortError)
await expectRequestAborted(this.server.lastReq)
})
it('destroys the request body when an error occurs', async function () {
const stream = Readable.from(infiniteIterator())
await expect(
abortOnceReceived(
signal =>
fetchStream(this.url('/hang'), {
method: 'POST',
body: stream,
signal,
}),
this.server
)
).to.be.rejectedWith(AbortError)
expect(stream.destroyed).to.be.true
})
it('detaches from signal on success', async function () {
const signal = AbortSignal.timeout(10_000)
for (let i = 0; i < 20; i++) {
const s = await fetchStream(this.url('/hello'), { signal })
expect(events.getEventListeners(signal, 'abort')).to.have.length(1)
await pipeline(s, fs.createWriteStream('/dev/null'))
expect(events.getEventListeners(signal, 'abort')).to.have.length(0)
}
})
it('detaches from signal on error', async function () {
const signal = AbortSignal.timeout(10_000)
for (let i = 0; i < 20; i++) {
try {
await fetchStream(this.url('/500'), { signal })
} catch (err) {
if (err instanceof RequestFailedError && err.response.status === 500)
continue
throw err
} finally {
expect(events.getEventListeners(signal, 'abort')).to.have.length(0)
}
}
})
})
describe('fetchNothing', function () {
it('closes the connection', async function () {
await fetchNothing(this.url('/large'))
await expectRequestAborted(this.server.lastReq)
})
it('aborts the request when the request body is destroyed before transfer', async function () {
const stream = Readable.from(infiniteIterator())
const promise = fetchNothing(this.url('/hang'), {
method: 'POST',
body: stream,
})
stream.destroy()
await expect(promise).to.be.rejectedWith(AbortError)
expect(this.server.lastReq).to.be.undefined
})
it('aborts the request when the request body is destroyed during transfer', async function () {
const stream = Readable.from(infiniteIterator())
// Note: this test won't work on `/hang`
const promise = fetchNothing(this.url('/sink'), {
method: 'POST',
body: stream,
})
await once(this.server.events, 'request-received')
stream.destroy()
await expect(promise).to.be.rejectedWith(AbortError)
await wait(80)
await expectRequestAborted(this.server.lastReq)
})
it("doesn't abort the request if the request body ends normally", async function () {
const stream = Readable.from('hello there')
await fetchNothing(this.url('/sink'), { method: 'POST', body: stream })
})
it('handles errors', async function () {
await expect(fetchNothing(this.url('/500'))).to.be.rejectedWith(
RequestFailedError
)
await expectRequestAborted(this.server.lastReq)
})
it('supports abort signals', async function () {
await expect(
abortOnceReceived(
signal => fetchNothing(this.url('/hang'), { signal }),
this.server
)
).to.be.rejectedWith(AbortError)
await expectRequestAborted(this.server.lastReq)
})
it('destroys the request body when an error occurs', async function () {
const stream = Readable.from(infiniteIterator())
await expect(
abortOnceReceived(
signal =>
fetchNothing(this.url('/hang'), {
method: 'POST',
body: stream,
signal,
}),
this.server
)
).to.be.rejectedWith(AbortError)
expect(stream.destroyed).to.be.true
})
})
describe('fetchString', function () {
it('returns a string', async function () {
const body = await fetchString(this.url('/hello'))
expect(body).to.equal('hello')
})
it('handles errors', async function () {
await expect(fetchString(this.url('/500'))).to.be.rejectedWith(
RequestFailedError
)
await expectRequestAborted(this.server.lastReq)
})
})
describe('fetchRedirect', function () {
it('returns the immediate redirect', async function () {
const body = await fetchRedirect(this.url('/redirect/1'))
expect(body).to.equal(this.url('/redirect/2'))
})
it('rejects status 200', async function () {
await expect(fetchRedirect(this.url('/hello'))).to.be.rejectedWith(
RequestFailedError
)
await expectRequestAborted(this.server.lastReq)
})
it('rejects empty redirect', async function () {
await expect(fetchRedirect(this.url('/redirect/empty-location')))
.to.be.rejectedWith(RequestFailedError)
.and.eventually.have.property('cause')
.and.to.have.property('message')
.to.equal('missing Location response header on 3xx response')
await expectRequestAborted(this.server.lastReq)
})
it('handles errors', async function () {
await expect(fetchRedirect(this.url('/500'))).to.be.rejectedWith(
RequestFailedError
)
await expectRequestAborted(this.server.lastReq)
})
})
describe('CustomHttpAgent', function () {
it('makes an http request successfully', async function () {
const agent = new CustomHttpAgent({ connectTimeout: 100 })
const body = await fetchString(this.url('/hello'), { agent })
expect(body).to.equal('hello')
})
it('times out when accessing a non-routable address', async function () {
const agent = new CustomHttpAgent({ connectTimeout: 10 })
await expect(fetchString('http://10.255.255.255/', { agent }))
.to.be.rejectedWith(FetchError)
.and.eventually.have.property('message')
.and.to.equal(
'request to http://10.255.255.255/ failed, reason: connect timeout'
)
})
})
describe('CustomHttpsAgent', function () {
it('makes an https request successfully', async function () {
const agent = new CustomHttpsAgent({
connectTimeout: 100,
ca: PUBLIC_CERT,
})
const body = await fetchString(this.httpsUrl('/hello'), { agent })
expect(body).to.equal('hello')
})
it('rejects an untrusted server', async function () {
const agent = new CustomHttpsAgent({
connectTimeout: 100,
})
await expect(fetchString(this.httpsUrl('/hello'), { agent }))
.to.be.rejectedWith(FetchError)
.and.eventually.have.property('code')
.and.to.equal('DEPTH_ZERO_SELF_SIGNED_CERT')
})
it('times out when accessing a non-routable address', async function () {
const agent = new CustomHttpsAgent({ connectTimeout: 10 })
await expect(fetchString('https://10.255.255.255/', { agent }))
.to.be.rejectedWith(FetchError)
.and.eventually.have.property('message')
.and.to.equal(
'request to https://10.255.255.255/ failed, reason: connect timeout'
)
})
})
})
async function streamToString(stream) {
let s = ''
for await (const chunk of stream) {
s += chunk
}
return s
}
async function* infiniteIterator() {
let i = 1
while (true) {
yield `chunk ${i++}\n`
}
}
/**
* @param {(signal: AbortSignal) => Promise<any>} func
* @param {TestServer} server
*/
async function abortOnceReceived(func, server) {
const controller = new AbortController()
const promise = func(controller.signal)
expect(events.getEventListeners(controller.signal, 'abort')).to.have.length(1)
await once(server.events, 'request-received')
controller.abort()
try {
return await promise
} finally {
expect(events.getEventListeners(controller.signal, 'abort')).to.have.length(
0
)
}
}
async function expectRequestAborted(req) {
if (!req.destroyed) {
try {
await once(req, 'close')
} catch (err) {
// `once` throws if req emits an 'error' event.
// We ignore `Error: aborted` when the request is aborted.
if (err.message !== 'aborted') {
throw err
}
}
}
expect(req.destroyed).to.be.true
}
const wait = ms => new Promise(resolve => setTimeout(resolve, ms))
@@ -1,139 +0,0 @@
const express = require('express')
const bodyParser = require('body-parser')
const { EventEmitter } = require('node:events')
const http = require('node:http')
const https = require('node:https')
const { promisify } = require('node:util')
class TestServer {
constructor() {
this.app = express()
this.events = new EventEmitter()
this.app.use(bodyParser.json())
this.app.use((req, res, next) => {
this.events.emit('request-received')
this.lastReq = req
next()
})
// Plain text endpoints
this.app.get('/hello', (req, res) => {
res.send('hello')
})
this.largePayload = 'x'.repeat(16 * 1024 * 1024)
this.app.get('/large', (req, res) => {
res.send(this.largePayload)
})
this.app.get('/204', (req, res) => {
res.status(204).end()
})
this.app.get('/empty', (req, res) => {
res.end()
})
this.app.get('/500', (req, res) => {
res.sendStatus(500)
})
this.app.post('/sink', (req, res) => {
req.on('data', () => {})
req.on('end', () => {
res.status(204).end()
})
})
// JSON endpoints
this.app.get('/json/hello', (req, res) => {
res.json({ msg: 'hello' })
})
this.app.post('/json/add', (req, res) => {
const { a, b } = req.body
res.json({ sum: a + b })
})
this.app.get('/json/500', (req, res) => {
res.status(500).json({ error: 'Internal server error' })
})
this.app.get('/json/basic-auth', (req, res) => {
const expectedAuth =
'Basic ' + Buffer.from('user:pass').toString('base64')
if (req.headers.authorization === expectedAuth) {
res.json({ key: 'verysecret' })
} else {
res.status(401).json({ error: 'unauthorized' })
}
})
this.app.post('/json/ignore-request', (req, res) => {
res.json({ msg: 'hello' })
})
// Never returns
this.app.get('/hang', (req, res) => {})
this.app.post('/hang', (req, res) => {})
// Redirect
this.app.get('/redirect/1', (req, res) => {
res.redirect('/redirect/2')
})
this.app.get('/redirect/2', (req, res) => {
res.send('body after redirect')
})
this.app.get('/redirect/empty-location', (req, res) => {
res.sendStatus(302)
})
}
start(port, httpsPort, httpsOptions) {
const startHttp = new Promise((resolve, reject) => {
this.server = http.createServer(this.app).listen(port, err => {
if (err) {
reject(err)
} else {
resolve()
}
})
})
const startHttps = new Promise((resolve, reject) => {
this.https_server = https
.createServer(httpsOptions, this.app)
.listen(httpsPort, err => {
if (err) {
reject(err)
} else {
resolve()
}
})
})
return Promise.all([startHttp, startHttps])
}
stop() {
const promises = []
if (this.server) {
const stopHttp = promisify(this.server.close).bind(this.server)
this.server.closeAllConnections()
promises.push(stopHttp())
}
if (this.https_server) {
const stopHttps = promisify(this.https_server.close).bind(
this.https_server
)
this.https_server.closeAllConnections()
promises.push(stopHttps())
}
return Promise.all(promises)
}
}
module.exports = { TestServer }
-4
View File
@@ -1,4 +0,0 @@
{
"extends": "../../tsconfig.backend.json",
"include": ["**/*.js", "**/*.cjs", "**/*.ts"]
}
+38
View File
@@ -0,0 +1,38 @@
{
"extends": [
"standard",
"prettier",
"prettier/standard"
],
"plugins": [
"mocha",
"chai-expect",
"chai-friendly"
],
"env": {
"mocha": true
},
"globals": {
"expect": true,
"define": true,
},
"settings": {
},
"rules": {
// Add some mocha specific rules
"mocha/handle-done-callback": "error",
"mocha/no-exclusive-tests": "error",
"mocha/no-global-tests": "error",
"mocha/no-identical-title": "error",
"mocha/no-nested-tests": "error",
"mocha/no-pending-tests": "error",
"mocha/no-skipped-tests": "error",
// Add some chai specific rules
"chai-expect/missing-assertion": "error",
"chai-expect/terminating-properties": "error",
// Swap the no-unused-expressions rule with a more chai-friendly one
"no-unused-expressions": 0,
"chai-friendly/no-unused-expressions": "error"
}
}
+4
View File
@@ -0,0 +1,4 @@
node_modules
.npmrc
Dockerfile
-13
View File
@@ -1,13 +0,0 @@
let reporterOptions = {}
if (process.env.CI) {
reporterOptions = {
reporter: require.resolve('mocha-multi-reporters'),
'reporter-options': ['configFile=./test/mocha-multi-reporters.cjs'],
}
}
const all = {
require: 'test/setup.js',
...reporterOptions,
}
module.exports = all
-1
View File
@@ -1 +0,0 @@
24.14.1
+4
View File
@@ -0,0 +1,4 @@
{
"semi": false,
"singleQuote": true
}
-9
View File
@@ -1,9 +0,0 @@
## v3.1.1
- Handle malformed requests in the req serializer
## v3.0.0
- Improve logging in Google Cloud Platform. Set environment variable `GCP_LOGGING=true` to enable.
This version of the metrics module only works with versions of the `@overleaf/metrics` module greater than v4.0.0
+3 -4
View File
@@ -1,10 +1,9 @@
logger
--dependencies=None
--docker-repos=gcr.io/overleaf-ops
--env-add=
--env-pass-through=
--esmock-loader=False
--is-library=True
--node-version=24.14.1
--package-name=@overleaf/logger
--pipeline-owner=32
--node-version=12.22.3
--public-repo=False
--script-version=3.11.0
-133
View File
@@ -1,133 +0,0 @@
const bunyan = require('bunyan')
/**
* When we copy log entry fields, omit some bunyan core fields that are not
* interesting, that have a special meaning in GCP, or that we will process
* separately.
*/
const ENTRY_FIELDS_TO_OMIT = [
'level',
'name',
'hostname',
'v',
'pid',
'msg',
'err',
'error',
'req',
'res',
]
/**
* Convert a bunyan log entry to a format that GCP understands
*/
function convertLogEntry(entry) {
const gcpEntry = omit(entry, ENTRY_FIELDS_TO_OMIT)
// Error information. In GCP, the stack trace goes in the message property.
// This enables the error reporting feature.
const err = entry.err || entry.error
if (err) {
if (err.info) {
Object.assign(gcpEntry, err.info)
}
if (err.code) {
gcpEntry.code = err.code
}
if (err.signal) {
gcpEntry.signal = err.signal
}
const stack = err.stack
if (stack && stack !== '(no stack)') {
gcpEntry.message = stack
} else if (err.message) {
gcpEntry.message = err.message
}
if (entry.name) {
gcpEntry.serviceContext = { service: entry.name }
}
}
// Log message
if (entry.msg) {
if (gcpEntry.message) {
// A message has already been extracted from the error. Keep the extra
// message in the msg property.
gcpEntry.msg = entry.msg
} else {
gcpEntry.message = entry.msg
}
}
// Severity
if (entry.level) {
gcpEntry.severity = bunyan.nameFromLevel[entry.level]
}
// HTTP request information
if (entry.req || entry.res || entry.responseTimeMs) {
const httpRequest = {}
if (entry.req) {
const req = entry.req
httpRequest.requestMethod = req.method
httpRequest.requestUrl = req.url
httpRequest.remoteIp = req.remoteAddress
if (req.headers) {
if (req.headers['content-length']) {
httpRequest.requestSize = parseInt(req.headers['content-length'], 10)
}
httpRequest.userAgent = req.headers['user-agent']
httpRequest.referer = req.headers.referer
}
}
if (entry.res) {
const res = entry.res
httpRequest.status = res.statusCode
if (res.headers && res.headers['content-length']) {
if (res.headers['content-length']) {
httpRequest.responseSize = parseInt(res.headers['content-length'], 10)
}
}
}
if (entry.responseTimeMs) {
const responseTimeSec = entry.responseTimeMs / 1000
httpRequest.latency = `${responseTimeSec}s`
}
gcpEntry.httpRequest = httpRequest
}
// Labels are indexed in GCP. We copy the project, doc and user ids to labels to enable fast filtering
const projectId =
gcpEntry.projectId ||
gcpEntry.project_id ||
(entry.req && entry.req.projectId)
const userId =
gcpEntry.userId || gcpEntry.user_id || (entry.req && entry.req.userId)
const docId =
gcpEntry.docId || gcpEntry.doc_id || (entry.req && entry.req.docId)
if (projectId || userId || docId) {
const labels = {}
if (projectId) {
labels.projectId = projectId
}
if (userId) {
labels.userId = userId
}
if (docId) {
labels.docId = docId
}
gcpEntry['logging.googleapis.com/labels'] = labels
}
return gcpEntry
}
function omit(obj, excludedFields) {
return Object.fromEntries(
Object.entries(obj).filter(([key]) => !excludedFields.includes(key))
)
}
module.exports = { convertLogEntry }
-60
View File
@@ -1,60 +0,0 @@
const { fetchString } = require('@overleaf/fetch-utils')
const fs = require('node:fs')
class LogLevelChecker {
constructor(logger, defaultLevel) {
this.logger = logger
this.defaultLevel = defaultLevel
}
start() {
// check for log level override on startup
this.checkLogLevel()
// re-check log level every minute
this.checkInterval = setInterval(this.checkLogLevel.bind(this), 1000 * 60)
this.checkInterval.unref()
}
stop() {
clearInterval(this.checkInterval)
}
async checkLogLevel() {
try {
const end = await this.getTracingEndTime()
if (end > Date.now()) {
this.logger.level('trace')
} else {
this.logger.level(this.defaultLevel)
}
} catch (e) {
this.logger.level(this.defaultLevel)
}
}
async getTracingEndTime() {
return 0
}
}
class FileLogLevelChecker extends LogLevelChecker {
async getTracingEndTime() {
const strEndTime = await fs.promises.readFile('/logging/tracingEndTime')
return parseInt(strEndTime, 10)
}
}
class GCEMetadataLogLevelChecker extends LogLevelChecker {
async getTracingEndTime() {
const options = {
headers: {
'Metadata-Flavor': 'Google',
},
}
const uri = `http://metadata.google.internal/computeMetadata/v1/project/attributes/${this.logger.fields.name}-setLogLevelEndTime`
const strEndTime = await fetchString(uri, options)
return parseInt(strEndTime, 10)
}
}
module.exports = { FileLogLevelChecker, GCEMetadataLogLevelChecker }

Some files were not shown because too many files have changed in this diff Show More