Compare commits
15
Commits
git-bridge-test
...
prod
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
45609de048 | ||
|
|
5b714873f3 | ||
|
|
eb74c4fced | ||
|
|
8eac611f70 | ||
|
|
90d9169c13 | ||
|
|
7cc21f0fd6 | ||
|
|
31971664ba | ||
|
|
ac11cc7505 | ||
|
|
57a1ce4f14 | ||
|
|
930f934e31 | ||
|
|
34d96c257f | ||
|
|
e693aa6019 | ||
|
|
33ed7487b8 | ||
|
|
e35c8e7a45 | ||
|
|
500bf04f91 |
@@ -331,6 +331,8 @@ jobs:
|
||||
value: "true"
|
||||
- name: OVERLEAF_LATEX_SHELL_ESCAPE
|
||||
value: "true"
|
||||
- name: OVERLEAF_ENABLE_GIT_SYNC
|
||||
value: "true"
|
||||
# (SMTP email vars are loaded below via envFrom.)
|
||||
# SMTP for password-reset / invite emails. All
|
||||
# OVERLEAF_EMAIL_* vars come from the optional 'verso-smtp'
|
||||
|
||||
@@ -42,6 +42,12 @@ server {
|
||||
font/woff2 woff2;
|
||||
application/pdf pdf;
|
||||
text/plain log blg aux stdout stderr txt;
|
||||
video/mp4 mp4;
|
||||
video/webm webm;
|
||||
video/ogg ogv ogg;
|
||||
audio/mpeg mp3;
|
||||
audio/ogg oga;
|
||||
audio/wav wav;
|
||||
}
|
||||
# handle output files for specific users
|
||||
location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/(.+)$ {
|
||||
|
||||
@@ -66,13 +66,14 @@ server {
|
||||
expires 1y;
|
||||
}
|
||||
|
||||
# handle output files for specific users
|
||||
location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ {
|
||||
# handle output files for specific users (all files, not just output.*)
|
||||
# Security: build ID is an unguessable random token, same model as output.html
|
||||
location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/.+ {
|
||||
proxy_pass http://127.0.0.1:8080; # clsi-nginx.conf
|
||||
proxy_http_version 1.1;
|
||||
}
|
||||
# handle output files for anonymous users
|
||||
location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ {
|
||||
location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/.+ {
|
||||
proxy_pass http://127.0.0.1:8080; # clsi-nginx.conf
|
||||
proxy_http_version 1.1;
|
||||
}
|
||||
|
||||
@@ -130,7 +130,11 @@ async function _startWatcher(compileName, options) {
|
||||
const timeout = options.timeout || 60_000
|
||||
|
||||
const absInput = Path.join(directory, mainFile)
|
||||
const absOutput = Path.join(directory, 'output.pdf')
|
||||
// Write to a staging path so that a concurrent recompile (triggered by file
|
||||
// changes detected while the current compile was running) can't truncate
|
||||
// output.pdf while CompileManager is copying it. We atomically rename to
|
||||
// output.pdf the moment "compiled successfully" is seen (see _onWatcherData).
|
||||
const absOutput = Path.join(directory, 'output-typst-snap.pdf')
|
||||
const env = { ...process.env, ...(environment || {}) }
|
||||
|
||||
logger.debug({ compileName, absInput }, 'starting typst watcher')
|
||||
@@ -259,6 +263,24 @@ function _onWatcherData(compileName, chunk) {
|
||||
entry.restartPending = true
|
||||
}
|
||||
|
||||
const compiledWithErrors = /compiled with errors/.test(line)
|
||||
|
||||
// Atomically rename the staging output to output.pdf RIGHT NOW, before
|
||||
// yielding the event loop. typst watch can trigger a new compile cycle
|
||||
// immediately after emitting this line (when file changes were detected
|
||||
// during the current compile), and that cycle would truncate the staging
|
||||
// file. Renaming here — before _finalizeCompile's 150 ms wait — ensures
|
||||
// we snapshot the complete PDF while it is still intact.
|
||||
// The rename promise is awaited in _finalizeCompile before resolving.
|
||||
let snapshotPromise = Promise.resolve()
|
||||
if (!compiledWithErrors) {
|
||||
const snapSrc = Path.join(entry.directory, 'output-typst-snap.pdf')
|
||||
const snapDst = Path.join(entry.directory, 'output.pdf')
|
||||
snapshotPromise = fs.promises.rename(snapSrc, snapDst).catch(err => {
|
||||
logger.warn({ err, compileName }, 'typst: failed to snapshot output.pdf')
|
||||
})
|
||||
}
|
||||
|
||||
// typst watch outputs the "[HH:MM:SS] compiled with errors" status
|
||||
// line FIRST, then the full diagnostics (file:line:col, code snippets)
|
||||
// AFTERWARDS. Enter post-done phase: keep accumulating into currentLines
|
||||
@@ -266,7 +288,8 @@ function _onWatcherData(compileName, chunk) {
|
||||
// cycle's COMPILE_START_RE arrives, whichever comes first).
|
||||
entry.doneResult = {
|
||||
preLines: entry.currentLines,
|
||||
compiledWithErrors: /compiled with errors/.test(line),
|
||||
compiledWithErrors,
|
||||
snapshotPromise,
|
||||
}
|
||||
entry.currentLines = []
|
||||
|
||||
@@ -281,20 +304,25 @@ function _onWatcherData(compileName, chunk) {
|
||||
|
||||
// Combines the pre-done lines (up to/including the status line) with any
|
||||
// post-done diagnostic lines and resolves all pending waiters.
|
||||
function _finalizeCompile(compileName) {
|
||||
async function _finalizeCompile(compileName) {
|
||||
const entry = WatchTable[compileName]
|
||||
if (!entry || !entry.doneResult) return
|
||||
|
||||
clearTimeout(entry.flushTimeout)
|
||||
entry.flushTimeout = null
|
||||
|
||||
const { preLines, compiledWithErrors } = entry.doneResult
|
||||
const { preLines, compiledWithErrors, snapshotPromise } = entry.doneResult
|
||||
entry.doneResult = null
|
||||
|
||||
// Merge: status line(s) first, then the post-done diagnostics.
|
||||
const allLines = preLines.concat(entry.currentLines)
|
||||
entry.currentLines = []
|
||||
|
||||
// Wait for the output-typst-snap.pdf → output.pdf rename to complete before
|
||||
// telling CompileManager the compile is done. On success this is typically
|
||||
// already resolved (rename starts synchronously in _onWatcherData).
|
||||
await snapshotPromise
|
||||
|
||||
_resolveAllPending(compileName, {
|
||||
stdout: allLines.join('\n'),
|
||||
compiledWithErrors,
|
||||
|
||||
@@ -81,12 +81,12 @@ async function pullFromGit(req, res) {
|
||||
const projectId = req.params.project_id
|
||||
try {
|
||||
const userId = SessionManager.getLoggedInUserId(req.session)
|
||||
const { remoteUrl, subPath } = await GitSyncHandler.getConfig(projectId)
|
||||
const { remoteUrl, subPath, branch } = await GitSyncHandler.getConfig(projectId)
|
||||
if (!remoteUrl) {
|
||||
return res.status(400).json({ error: 'No git remote configured for this project' })
|
||||
}
|
||||
logger.debug({ projectId }, 'git sync: starting pull')
|
||||
await GitSyncHandler.pullFromRemote(projectId, remoteUrl, subPath, userId)
|
||||
await GitSyncHandler.pullFromRemote(projectId, remoteUrl, subPath, branch, userId)
|
||||
res.sendStatus(204)
|
||||
} catch (err) {
|
||||
logger.warn({ err, projectId }, 'git sync: pull failed')
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
import { mkdtemp, mkdir, writeFile, rm, readdir, readFile, access } from 'node:fs/promises'
|
||||
import { mkdir, writeFile, rm, readdir, readFile, access } from 'node:fs/promises'
|
||||
import pLimit from 'p-limit'
|
||||
import { createWriteStream } from 'node:fs'
|
||||
import { join, dirname, extname, basename } from 'node:path'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { spawn } from 'node:child_process'
|
||||
import { pipeline } from 'node:stream/promises'
|
||||
import logger from '@overleaf/logger'
|
||||
import Settings from '@overleaf/settings'
|
||||
import ProjectEntityHandler from '../Project/ProjectEntityHandler.mjs'
|
||||
import ProjectEntityUpdateHandler from '../Project/ProjectEntityUpdateHandler.mjs'
|
||||
import DocumentUpdaterHandler from '../DocumentUpdater/DocumentUpdaterHandler.mjs'
|
||||
import HistoryManager from '../History/HistoryManager.mjs'
|
||||
import ClsiManager from '../Compile/ClsiManager.mjs'
|
||||
import { Project } from '../../models/Project.mjs'
|
||||
@@ -20,6 +21,13 @@ const EDITABLE_FILENAMES = new Set(
|
||||
(Settings.editableFilenames || []).map(n => n.toLowerCase())
|
||||
)
|
||||
|
||||
// Data root is one level above the dumpFolder (e.g. /var/lib/overleaf/data)
|
||||
const GIT_CACHE_ROOT = join(dirname(Settings.path.dumpFolder), 'git-sync-cache')
|
||||
|
||||
function repoCachePath(projectId) {
|
||||
return join(GIT_CACHE_ROOT, projectId.toString())
|
||||
}
|
||||
|
||||
function isTextFile(filename) {
|
||||
const ext = extname(filename).toLowerCase()
|
||||
return TEXT_EXTENSIONS.has(ext) || EDITABLE_FILENAMES.has(filename.toLowerCase())
|
||||
@@ -59,6 +67,24 @@ async function spawnGit(args, cwd) {
|
||||
})
|
||||
}
|
||||
|
||||
async function spawnGitOutput(args, cwd) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const proc = spawn('git', args, {
|
||||
cwd,
|
||||
env: { ...process.env, GIT_TERMINAL_PROMPT: '0' },
|
||||
})
|
||||
let stdout = ''
|
||||
let stderr = ''
|
||||
proc.stdout?.on('data', d => (stdout += d.toString()))
|
||||
proc.stderr?.on('data', d => (stderr += d.toString()))
|
||||
proc.on('close', code => {
|
||||
if (code !== 0) reject(new Error(`git ${args[0]} exited ${code}: ${stderr.trim()}`))
|
||||
else resolve(stdout.trim())
|
||||
})
|
||||
proc.on('error', reject)
|
||||
})
|
||||
}
|
||||
|
||||
async function setConfig(projectId, { remoteUrl, subPath, pdfPath, pushFiles, pushPdf, branch }) {
|
||||
await Project.updateOne(
|
||||
{ _id: new ObjectId(projectId) },
|
||||
@@ -94,150 +120,222 @@ async function getConfig(projectId) {
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure a persistent cached repo exists and is connected to the right remote.
|
||||
// Returns the repo directory path.
|
||||
async function ensureRepo(projectId, remoteUrl, branch) {
|
||||
const repoDir = repoCachePath(projectId)
|
||||
await mkdir(repoDir, { recursive: true })
|
||||
|
||||
let needsInit = false
|
||||
try {
|
||||
const currentRemote = await spawnGitOutput(['remote', 'get-url', 'origin'], repoDir)
|
||||
if (currentRemote !== remoteUrl) {
|
||||
// Remote URL changed (e.g. new token) — update it in place
|
||||
await spawnGit(['remote', 'set-url', 'origin', remoteUrl], repoDir)
|
||||
}
|
||||
} catch {
|
||||
needsInit = true
|
||||
}
|
||||
|
||||
if (needsInit) {
|
||||
await rm(repoDir, { recursive: true, force: true })
|
||||
await mkdir(repoDir, { recursive: true })
|
||||
await spawnGit(['init'], repoDir)
|
||||
await spawnGit(['symbolic-ref', 'HEAD', `refs/heads/${branch}`], repoDir)
|
||||
await spawnGit(['config', 'user.email', 'verso-sync@localhost'], repoDir)
|
||||
await spawnGit(['config', 'user.name', 'Verso Sync'], repoDir)
|
||||
await spawnGit(['remote', 'add', 'origin', remoteUrl], repoDir)
|
||||
} else {
|
||||
await spawnGit(['config', 'user.email', 'verso-sync@localhost'], repoDir)
|
||||
await spawnGit(['config', 'user.name', 'Verso Sync'], repoDir)
|
||||
// Remove promisor config if present from a prior version — it causes git to lazy-fetch
|
||||
// missing blobs during commit (connectivity check), which we don't want.
|
||||
// --filter=blob:none on fetch works independently of this config.
|
||||
for (const key of ['remote.origin.partialclonefilter', 'remote.origin.promisor']) {
|
||||
try { await spawnGit(['config', '--unset', key], repoDir) } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
return repoDir
|
||||
}
|
||||
|
||||
async function pushToRemote(
|
||||
projectId,
|
||||
remoteUrl,
|
||||
subPath,
|
||||
{ pdfPath, pdfBuildId, pdfClsiServerId, userId, pushFiles = true, pushPdf = true, branch = 'main' } = {}
|
||||
) {
|
||||
const tmpDir = await mkdtemp(join(tmpdir(), 'verso-git-'))
|
||||
logger.debug({ projectId, tmpDir, subPath, pushFiles, pushPdf, branch }, 'git sync: starting push')
|
||||
try {
|
||||
const repoDir = join(tmpDir, 'repo')
|
||||
await mkdir(repoDir)
|
||||
logger.debug({ projectId, subPath, pushFiles, pushPdf, branch }, 'git sync: starting push')
|
||||
|
||||
// Init, connect to remote, fetch existing state so files outside our
|
||||
// managed area are preserved in the push.
|
||||
await spawnGit(['init'], repoDir)
|
||||
await spawnGit(['symbolic-ref', 'HEAD', `refs/heads/${branch}`], repoDir)
|
||||
await spawnGit(['config', 'user.email', 'verso-sync@localhost'], repoDir)
|
||||
await spawnGit(['config', 'user.name', 'Verso Sync'], repoDir)
|
||||
await spawnGit(['remote', 'add', 'origin', remoteUrl], repoDir)
|
||||
try {
|
||||
await spawnGit(['fetch', '--depth', '1', 'origin', branch], repoDir)
|
||||
await spawnGit(['reset', '--hard', 'FETCH_HEAD'], repoDir)
|
||||
} catch {
|
||||
// Remote branch doesn't exist yet (first push to empty repo) — proceed
|
||||
}
|
||||
|
||||
const fileRoot = subPath ? join(repoDir, subPath) : repoDir
|
||||
|
||||
if (pushFiles) {
|
||||
// Clear the managed area so files deleted in Verso are removed from git.
|
||||
// With a subPath this leaves everything outside the subPath untouched.
|
||||
// Without a subPath the entire repo is considered Verso-owned.
|
||||
if (subPath) {
|
||||
await rm(fileRoot, { recursive: true, force: true })
|
||||
await mkdir(fileRoot, { recursive: true })
|
||||
} else {
|
||||
const entries = await readdir(repoDir, { withFileTypes: true })
|
||||
for (const entry of entries) {
|
||||
if (entry.name === '.git') continue
|
||||
await rm(join(repoDir, entry.name), { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
const docs = await ProjectEntityHandler.promises.getAllDocs(projectId)
|
||||
for (const [path, doc] of Object.entries(docs)) {
|
||||
const rel = path.startsWith('/') ? path.slice(1) : path
|
||||
const dest = join(fileRoot, rel)
|
||||
await mkdir(dirname(dest), { recursive: true })
|
||||
await writeFile(dest, doc.lines.join('\n'))
|
||||
}
|
||||
|
||||
const files = await ProjectEntityHandler.promises.getAllFiles(projectId)
|
||||
for (const [path, file] of Object.entries(files)) {
|
||||
const rel = path.startsWith('/') ? path.slice(1) : path
|
||||
const dest = join(fileRoot, rel)
|
||||
await mkdir(dirname(dest), { recursive: true })
|
||||
const { stream } =
|
||||
await HistoryManager.promises.requestBlobWithProjectId(
|
||||
projectId,
|
||||
file.hash
|
||||
)
|
||||
await pipeline(stream, createWriteStream(dest))
|
||||
}
|
||||
}
|
||||
|
||||
// PDF path is always relative to repo root, independent of subPath
|
||||
if (pushPdf && pdfPath && pdfBuildId) {
|
||||
const pdfStream = await ClsiManager.promises.getOutputFileStream(
|
||||
projectId,
|
||||
userId,
|
||||
pdfClsiServerId ?? undefined,
|
||||
pdfBuildId,
|
||||
'output.pdf'
|
||||
)
|
||||
const pdfDest = join(repoDir, pdfPath)
|
||||
await mkdir(dirname(pdfDest), { recursive: true })
|
||||
await pipeline(pdfStream, createWriteStream(pdfDest))
|
||||
logger.debug({ projectId }, 'git sync: PDF included')
|
||||
}
|
||||
|
||||
await spawnGit(['add', '-A'], repoDir)
|
||||
await spawnGit(['commit', '--allow-empty', '-m', 'Verso sync'], repoDir)
|
||||
await spawnGit(['push', '--force', 'origin', `HEAD:${branch}`], repoDir)
|
||||
|
||||
logger.debug({ projectId }, 'git sync: push complete')
|
||||
} finally {
|
||||
await rm(tmpDir, { recursive: true, force: true })
|
||||
// Flush in-memory document edits to MongoDB before reading, so getAllDocs returns current content.
|
||||
// Without this, edits made in the editor may not yet be persisted and would be silently skipped.
|
||||
if (pushFiles) {
|
||||
await DocumentUpdaterHandler.promises.flushProjectToMongo(projectId)
|
||||
}
|
||||
|
||||
// Use a persistent repo so git's delta mechanism works across pushes.
|
||||
// On first push all objects are new; on subsequent pushes only changed
|
||||
// objects are sent — git handles the diff automatically.
|
||||
const repoDir = await ensureRepo(projectId, remoteUrl, branch)
|
||||
|
||||
// Fetch remote state (trees only, latest commit only) to preserve files outside the managed area.
|
||||
// Skipped when remote HEAD already matches local HEAD (common case: re-push without external changes).
|
||||
try {
|
||||
let needsFetch = true
|
||||
try {
|
||||
const remoteRef = await spawnGitOutput(['ls-remote', 'origin', branch], repoDir)
|
||||
const remoteHead = remoteRef.split(/\s/)[0].trim()
|
||||
const localHead = await spawnGitOutput(['rev-parse', 'HEAD'], repoDir)
|
||||
needsFetch = remoteHead !== localHead
|
||||
} catch {
|
||||
// Can't compare — proceed with fetch
|
||||
}
|
||||
if (needsFetch) {
|
||||
// --depth=1 --filter=blob:none: only the latest commit's tree objects, no blobs
|
||||
await spawnGit(['fetch', '--filter=blob:none', '--depth=1', 'origin', branch], repoDir)
|
||||
// --mixed: moves HEAD to FETCH_HEAD + updates index, no working tree change.
|
||||
// This makes FETCH_HEAD the parent of our next commit so the diff only shows subPath changes.
|
||||
await spawnGit(['reset', '--mixed', 'FETCH_HEAD'], repoDir)
|
||||
}
|
||||
} catch {
|
||||
// Empty remote or first push — proceed with empty index
|
||||
}
|
||||
|
||||
const fileRoot = subPath ? join(repoDir, subPath) : repoDir
|
||||
|
||||
if (pushFiles) {
|
||||
// Clear the managed area so files deleted in Verso are removed from git.
|
||||
// With a subPath this leaves everything outside the subPath untouched.
|
||||
if (subPath) {
|
||||
await rm(fileRoot, { recursive: true, force: true })
|
||||
await mkdir(fileRoot, { recursive: true })
|
||||
} else {
|
||||
const entries = await readdir(repoDir, { withFileTypes: true })
|
||||
for (const entry of entries) {
|
||||
if (entry.name === '.git') continue
|
||||
await rm(join(repoDir, entry.name), { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
const docs = await ProjectEntityHandler.promises.getAllDocs(projectId)
|
||||
for (const [path, doc] of Object.entries(docs)) {
|
||||
const rel = path.startsWith('/') ? path.slice(1) : path
|
||||
const dest = join(fileRoot, rel)
|
||||
await mkdir(dirname(dest), { recursive: true })
|
||||
await writeFile(dest, doc.lines.join('\n'))
|
||||
}
|
||||
|
||||
const files = await ProjectEntityHandler.promises.getAllFiles(projectId)
|
||||
for (const [path, file] of Object.entries(files)) {
|
||||
const rel = path.startsWith('/') ? path.slice(1) : path
|
||||
const dest = join(fileRoot, rel)
|
||||
await mkdir(dirname(dest), { recursive: true })
|
||||
const { stream } =
|
||||
await HistoryManager.promises.requestBlobWithProjectId(
|
||||
projectId,
|
||||
file.hash
|
||||
)
|
||||
await pipeline(stream, createWriteStream(dest))
|
||||
}
|
||||
}
|
||||
|
||||
// PDF path is always relative to repo root, independent of subPath
|
||||
if (pushPdf && pdfPath && pdfBuildId) {
|
||||
const pdfStream = await ClsiManager.promises.getOutputFileStream(
|
||||
projectId,
|
||||
userId,
|
||||
pdfClsiServerId ?? undefined,
|
||||
pdfBuildId,
|
||||
'output.pdf'
|
||||
)
|
||||
const pdfDest = join(repoDir, pdfPath)
|
||||
await mkdir(dirname(pdfDest), { recursive: true })
|
||||
await pipeline(pdfStream, createWriteStream(pdfDest))
|
||||
logger.debug({ projectId }, 'git sync: PDF included')
|
||||
}
|
||||
|
||||
// Stage changes. For subPath, explicitly clear the index entries first then re-add from disk.
|
||||
// This bypasses git's cached-tree extension which can cause modified files to be silently
|
||||
// skipped by git add -A when the directory mtime hasn't changed since the last checkout.
|
||||
if (pushFiles) {
|
||||
if (subPath) {
|
||||
await spawnGit(['rm', '-rf', '--cached', '--ignore-unmatch', '--', subPath], repoDir)
|
||||
await spawnGit(['add', '--', subPath], repoDir)
|
||||
} else {
|
||||
await spawnGit(['add', '-A'], repoDir)
|
||||
}
|
||||
}
|
||||
if (pushPdf && pdfPath && pdfBuildId) {
|
||||
await spawnGit(['add', '--', pdfPath], repoDir)
|
||||
}
|
||||
await spawnGit(['commit', '--allow-empty', '-m', 'Verso sync'], repoDir)
|
||||
await spawnGit(['push', '--force', 'origin', `HEAD:${branch}`], repoDir)
|
||||
|
||||
logger.debug({ projectId }, 'git sync: push complete')
|
||||
// Note: repoDir is intentionally not deleted — it persists as the cache
|
||||
// for the next push so git can compute a proper delta.
|
||||
}
|
||||
|
||||
async function pullFromRemote(projectId, remoteUrl, subPath, userId) {
|
||||
const tmpDir = await mkdtemp(join(tmpdir(), 'verso-pull-'))
|
||||
logger.debug({ projectId, subPath }, 'git sync: cloning remote for pull')
|
||||
try {
|
||||
await spawnGit(['clone', '--depth', '1', '--', remoteUrl, 'repo'], tmpDir)
|
||||
async function pullFromRemote(projectId, remoteUrl, subPath, branch, userId) {
|
||||
logger.debug({ projectId, subPath }, 'git sync: fetching remote for pull')
|
||||
|
||||
const repoDir = join(tmpDir, 'repo')
|
||||
const fileRoot = subPath ? join(repoDir, subPath) : repoDir
|
||||
const repoDir = await ensureRepo(projectId, remoteUrl, branch)
|
||||
// depth=1 + blob filter: only the latest commit's tree objects, no blobs upfront
|
||||
await spawnGit(['fetch', '--filter=blob:none', '--depth=1', 'origin', branch], repoDir)
|
||||
|
||||
// Verify the subdirectory exists in the repo
|
||||
try {
|
||||
await access(fileRoot)
|
||||
} catch {
|
||||
throw new Error(
|
||||
subPath
|
||||
? `Subdirectory "${subPath}" not found in the repository`
|
||||
: 'Repository is empty'
|
||||
)
|
||||
}
|
||||
|
||||
const allFiles = await walkDir(fileRoot)
|
||||
logger.debug({ projectId, count: allFiles.length }, 'git sync: importing files from remote')
|
||||
|
||||
for (const absPath of allFiles) {
|
||||
// Path relative to fileRoot → Verso project path (leading slash required)
|
||||
const rel = absPath.slice(fileRoot.length)
|
||||
const projectPath = rel.startsWith('/') ? rel : `/${rel}`
|
||||
|
||||
if (isTextFile(basename(absPath))) {
|
||||
const content = await readFile(absPath, 'utf8')
|
||||
const lines = content.split('\n')
|
||||
await ProjectEntityUpdateHandler.promises.upsertDocWithPath(
|
||||
projectId,
|
||||
projectPath,
|
||||
lines,
|
||||
'git-sync',
|
||||
userId
|
||||
)
|
||||
} else {
|
||||
await ProjectEntityUpdateHandler.promises.upsertFileWithPath(
|
||||
projectId,
|
||||
projectPath,
|
||||
absPath,
|
||||
null,
|
||||
userId,
|
||||
'git-sync'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug({ projectId }, 'git sync: pull complete')
|
||||
} finally {
|
||||
await rm(tmpDir, { recursive: true, force: true })
|
||||
if (subPath) {
|
||||
// Checkout only the subPath directory — triggers lazy blob fetch for those files only
|
||||
await spawnGit(['checkout', 'FETCH_HEAD', '--', subPath], repoDir)
|
||||
} else {
|
||||
// Full reset — all blobs downloaded (unavoidable when no subPath)
|
||||
await spawnGit(['reset', '--hard', 'FETCH_HEAD'], repoDir)
|
||||
}
|
||||
|
||||
const fileRoot = subPath ? join(repoDir, subPath) : repoDir
|
||||
|
||||
try {
|
||||
await access(fileRoot)
|
||||
} catch {
|
||||
throw new Error(
|
||||
subPath
|
||||
? `Subdirectory "${subPath}" not found in the repository`
|
||||
: 'Repository is empty'
|
||||
)
|
||||
}
|
||||
|
||||
const allFiles = await walkDir(fileRoot)
|
||||
logger.debug({ projectId, count: allFiles.length }, 'git sync: importing files from remote')
|
||||
|
||||
const limit = pLimit(5)
|
||||
await Promise.all(
|
||||
allFiles.map(absPath =>
|
||||
limit(async () => {
|
||||
const rel = absPath.slice(fileRoot.length)
|
||||
const projectPath = rel.startsWith('/') ? rel : `/${rel}`
|
||||
if (isTextFile(basename(absPath))) {
|
||||
const content = await readFile(absPath, 'utf8')
|
||||
await ProjectEntityUpdateHandler.promises.upsertDocWithPath(
|
||||
projectId,
|
||||
projectPath,
|
||||
content.split('\n'),
|
||||
'git-sync',
|
||||
userId
|
||||
)
|
||||
} else {
|
||||
await ProjectEntityUpdateHandler.promises.upsertFileWithPath(
|
||||
projectId,
|
||||
projectPath,
|
||||
absPath,
|
||||
null,
|
||||
userId,
|
||||
'git-sync'
|
||||
)
|
||||
}
|
||||
})
|
||||
)
|
||||
)
|
||||
|
||||
logger.debug({ projectId }, 'git sync: pull complete')
|
||||
}
|
||||
|
||||
export default { setConfig, getConfig, pushToRemote, pullFromRemote }
|
||||
|
||||
@@ -1044,7 +1044,7 @@
|
||||
"git_bridge_modal_learn_more_about_authentication_tokens": "Learn more about Git integration authentication tokens.",
|
||||
"git_bridge_modal_read_only": "<strong>You have read-only access to this project.</strong> This means you can pull from __appName__ but you can’t push any changes you make back to this project.",
|
||||
"git_sync": "Git Sync",
|
||||
"git_sync_description": "Force-push your project files to an external git repository. Include your auth token in the URL (e.g. https://user:token@github.com/org/repo.git).",
|
||||
"git_sync_description": "Sync your project with an external git repository. Include your auth token in the URL (e.g. https://user:token@gitea.example.com/org/repo.git).",
|
||||
"git_sync_pull_now": "Pull from git",
|
||||
"git_sync_pull_success": "Project updated from git remote successfully.",
|
||||
"git_sync_pulling": "Pulling…",
|
||||
|
||||
@@ -1047,7 +1047,7 @@
|
||||
"git_bridge_modal_learn_more_about_authentication_tokens": "Apprenez-en davantage sur les jetons d’authentification de l’intégration Git.",
|
||||
"git_bridge_modal_read_only": "<strong>Vous disposez d'un accès en lecture seule à ce projet.</strong> Cela signifie que vous pouvez extraire des données de __appName__, mais que vous ne pouvez pas transférer les modifications que vous apportez à ce projet.",
|
||||
"git_sync": "Synchronisation Git",
|
||||
"git_sync_description": "Pousse de force les fichiers du projet vers un dépôt git externe. Incluez votre jeton d'authentification dans l'URL (ex. https://user:token@github.com/org/repo.git).",
|
||||
"git_sync_description": "Synchronisez votre projet avec un dépôt git externe. Incluez votre jeton d'authentification dans l'URL (ex. https://user:token@gitea.example.com/org/repo.git).",
|
||||
"git_sync_pull_now": "Tirer depuis git",
|
||||
"git_sync_pull_success": "Projet mis à jour depuis le dépôt git avec succès.",
|
||||
"git_sync_pulling": "Récupération…",
|
||||
|
||||
Reference in New Issue
Block a user