feat(git-sync): add pull from remote (2-way sync)
Build and Deploy Verso / deploy (push) Has been cancelled
Build and Deploy Verso / deploy (push) Has been cancelled
POST /project/:id/git-sync/pull clones the configured remote at depth 1, walks all files under the configured subPath, and upserts each into the Verso project using upsertDocWithPath (text) or upsertFileWithPath (binary), with full folder creation via mkdirp. The .git directory is skipped. Pull is additive/update-only — no Verso entities are deleted. Text vs binary classification uses Settings.textExtensions (same list the editor uses for file uploads), so .typ, .tex, .md, .yml etc. all become editable docs while images and PDFs stay as files. Frontend: "Pull from git" button added alongside "Push now". Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
d5de6550d4
commit
e1533d979f
@@ -69,8 +69,28 @@ async function getGitSyncConfig(req, res) {
|
||||
})
|
||||
}
|
||||
|
||||
async function pullFromGit(req, res) {
|
||||
const projectId = req.params.project_id
|
||||
const userId = req.session.user._id
|
||||
|
||||
try {
|
||||
const { remoteUrl, subPath } = await GitSyncHandler.getConfig(projectId)
|
||||
if (!remoteUrl) {
|
||||
return res.status(400).json({ error: 'No git remote configured for this project' })
|
||||
}
|
||||
|
||||
logger.debug({ projectId }, 'git sync: starting pull')
|
||||
await GitSyncHandler.pullFromRemote(projectId, remoteUrl, subPath, userId)
|
||||
res.sendStatus(204)
|
||||
} catch (err) {
|
||||
logger.warn({ err, projectId }, 'git sync: pull failed')
|
||||
res.status(500).json({ error: err.message })
|
||||
}
|
||||
}
|
||||
|
||||
export default {
|
||||
configureGitSync: expressify(configureGitSync),
|
||||
pushToGit: expressify(pushToGit),
|
||||
pullFromGit: expressify(pullFromGit),
|
||||
getGitSyncConfig: expressify(getGitSyncConfig),
|
||||
}
|
||||
|
||||
@@ -1,16 +1,45 @@
|
||||
import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises'
|
||||
import { mkdtemp, mkdir, writeFile, rm, readdir, readFile, access } from 'node:fs/promises'
|
||||
import { createWriteStream } from 'node:fs'
|
||||
import { join, dirname } from 'node:path'
|
||||
import { join, dirname, extname, basename } from 'node:path'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { spawn } from 'node:child_process'
|
||||
import { pipeline } from 'node:stream/promises'
|
||||
import logger from '@overleaf/logger'
|
||||
import Settings from '@overleaf/settings'
|
||||
import ProjectEntityHandler from '../Project/ProjectEntityHandler.mjs'
|
||||
import ProjectEntityUpdateHandler from '../Project/ProjectEntityUpdateHandler.mjs'
|
||||
import HistoryManager from '../History/HistoryManager.mjs'
|
||||
import ClsiManager from '../Compile/ClsiManager.mjs'
|
||||
import { Project } from '../../models/Project.mjs'
|
||||
import { ObjectId } from '../../infrastructure/mongodb.mjs'
|
||||
|
||||
const TEXT_EXTENSIONS = new Set(
|
||||
Settings.textExtensions.map(ext => `.${ext}`)
|
||||
)
|
||||
const EDITABLE_FILENAMES = new Set(
|
||||
(Settings.editableFilenames || []).map(n => n.toLowerCase())
|
||||
)
|
||||
|
||||
function isTextFile(filename) {
|
||||
const ext = extname(filename).toLowerCase()
|
||||
return TEXT_EXTENSIONS.has(ext) || EDITABLE_FILENAMES.has(filename.toLowerCase())
|
||||
}
|
||||
|
||||
async function walkDir(dir) {
|
||||
const entries = await readdir(dir, { withFileTypes: true })
|
||||
const files = []
|
||||
for (const entry of entries) {
|
||||
if (entry.name === '.git') continue
|
||||
const fullPath = join(dir, entry.name)
|
||||
if (entry.isDirectory()) {
|
||||
files.push(...(await walkDir(fullPath)))
|
||||
} else {
|
||||
files.push(fullPath)
|
||||
}
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
async function spawnGit(args, cwd) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const proc = spawn('git', args, {
|
||||
@@ -120,4 +149,60 @@ async function pushToRemote(
|
||||
}
|
||||
}
|
||||
|
||||
export default { setConfig, getConfig, pushToRemote }
|
||||
async function pullFromRemote(projectId, remoteUrl, subPath, userId) {
|
||||
const tmpDir = await mkdtemp(join(tmpdir(), 'verso-pull-'))
|
||||
logger.debug({ projectId, subPath }, 'git sync: cloning remote for pull')
|
||||
try {
|
||||
await spawnGit(['clone', '--depth', '1', '--', remoteUrl, 'repo'], tmpDir)
|
||||
|
||||
const repoDir = join(tmpDir, 'repo')
|
||||
const fileRoot = subPath ? join(repoDir, subPath) : repoDir
|
||||
|
||||
// Verify the subdirectory exists in the repo
|
||||
try {
|
||||
await access(fileRoot)
|
||||
} catch {
|
||||
throw new Error(
|
||||
subPath
|
||||
? `Subdirectory "${subPath}" not found in the repository`
|
||||
: 'Repository is empty'
|
||||
)
|
||||
}
|
||||
|
||||
const allFiles = await walkDir(fileRoot)
|
||||
logger.debug({ projectId, count: allFiles.length }, 'git sync: importing files from remote')
|
||||
|
||||
for (const absPath of allFiles) {
|
||||
// Path relative to fileRoot → Verso project path (leading slash required)
|
||||
const rel = absPath.slice(fileRoot.length)
|
||||
const projectPath = rel.startsWith('/') ? rel : `/${rel}`
|
||||
|
||||
if (isTextFile(basename(absPath))) {
|
||||
const content = await readFile(absPath, 'utf8')
|
||||
const lines = content.split('\n')
|
||||
await ProjectEntityUpdateHandler.promises.upsertDocWithPath(
|
||||
projectId,
|
||||
projectPath,
|
||||
lines,
|
||||
'git-sync',
|
||||
userId
|
||||
)
|
||||
} else {
|
||||
await ProjectEntityUpdateHandler.promises.upsertFileWithPath(
|
||||
projectId,
|
||||
projectPath,
|
||||
absPath,
|
||||
null,
|
||||
userId,
|
||||
'git-sync'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug({ projectId }, 'git sync: pull complete')
|
||||
} finally {
|
||||
await rm(tmpDir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
export default { setConfig, getConfig, pushToRemote, pullFromRemote }
|
||||
|
||||
@@ -622,6 +622,12 @@ async function initialize(webRouter, privateApiRouter, publicApiRouter) {
|
||||
AuthorizationMiddleware.ensureUserCanWriteProjectContent,
|
||||
GitSyncController.pushToGit
|
||||
)
|
||||
webRouter.post(
|
||||
'/project/:project_id/git-sync/pull',
|
||||
AuthenticationController.requireLogin(),
|
||||
AuthorizationMiddleware.ensureUserCanWriteProjectContent,
|
||||
GitSyncController.pullFromGit
|
||||
)
|
||||
|
||||
webRouter.post(
|
||||
'/project/:Project_id/compile',
|
||||
|
||||
@@ -796,6 +796,9 @@
|
||||
"git_bridge_modal_read_only": "",
|
||||
"git_sync": "",
|
||||
"git_sync_description": "",
|
||||
"git_sync_pull_now": "",
|
||||
"git_sync_pull_success": "",
|
||||
"git_sync_pulling": "",
|
||||
"git_sync_push_now": "",
|
||||
"git_sync_pushing": "",
|
||||
"git_sync_push_success": "",
|
||||
|
||||
@@ -11,7 +11,7 @@ import { FormCheck } from 'react-bootstrap'
|
||||
import MaterialIcon from '@/shared/components/material-icon'
|
||||
import OLNotification from '@/shared/components/ol/ol-notification'
|
||||
|
||||
type Action = 'save' | 'push'
|
||||
type Action = 'save' | 'push' | 'pull'
|
||||
type Status = 'idle' | 'busy' | 'success' | 'error'
|
||||
|
||||
const projectId = getMeta('ol-project_id')
|
||||
@@ -96,6 +96,20 @@ export default function GitSyncWidget() {
|
||||
doPush(pdfFile?.build, clsiServerId)
|
||||
}
|
||||
|
||||
async function handlePull() {
|
||||
setLastAction('pull')
|
||||
setStatus('busy')
|
||||
setErrorMsg('')
|
||||
try {
|
||||
await saveConfig()
|
||||
await postJSON(`/project/${projectId}/git-sync/pull`, { body: {} })
|
||||
setStatus('success')
|
||||
} catch (err: any) {
|
||||
setStatus('error')
|
||||
setErrorMsg(err?.data?.error ?? String(err))
|
||||
}
|
||||
}
|
||||
|
||||
function handleAutoPushToggle(e: React.ChangeEvent<HTMLInputElement>) {
|
||||
const checked = e.target.checked
|
||||
setAutoPush(checked)
|
||||
@@ -179,6 +193,17 @@ export default function GitSyncWidget() {
|
||||
>
|
||||
{t('git_sync_push_now')}
|
||||
</OLButton>
|
||||
<OLButton
|
||||
type="button"
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
onClick={handlePull}
|
||||
disabled={isBusy || !remoteUrl.trim()}
|
||||
isLoading={isBusy && lastAction === 'pull'}
|
||||
loadingLabel={t('git_sync_pulling')}
|
||||
>
|
||||
{t('git_sync_pull_now')}
|
||||
</OLButton>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
@@ -188,7 +213,9 @@ export default function GitSyncWidget() {
|
||||
content={
|
||||
lastAction === 'push'
|
||||
? t('git_sync_push_success')
|
||||
: t('git_sync_saved')
|
||||
: lastAction === 'pull'
|
||||
? t('git_sync_pull_success')
|
||||
: t('git_sync_saved')
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
@@ -1045,6 +1045,9 @@
|
||||
"git_bridge_modal_read_only": "<strong>You have read-only access to this project.</strong> This means you can pull from __appName__ but you can’t push any changes you make back to this project.",
|
||||
"git_sync": "Git Sync",
|
||||
"git_sync_description": "Force-push your project files to an external git repository. Include your auth token in the URL (e.g. https://user:token@github.com/org/repo.git).",
|
||||
"git_sync_pull_now": "Pull from git",
|
||||
"git_sync_pull_success": "Project updated from git remote successfully.",
|
||||
"git_sync_pulling": "Pulling…",
|
||||
"git_sync_push_now": "Push now",
|
||||
"git_sync_pushing": "Pushing…",
|
||||
"git_sync_push_success": "Project pushed to git remote successfully.",
|
||||
|
||||
@@ -1048,6 +1048,9 @@
|
||||
"git_bridge_modal_read_only": "<strong>Vous disposez d'un accès en lecture seule à ce projet.</strong> Cela signifie que vous pouvez extraire des données de __appName__, mais que vous ne pouvez pas transférer les modifications que vous apportez à ce projet.",
|
||||
"git_sync": "Synchronisation Git",
|
||||
"git_sync_description": "Pousse de force les fichiers du projet vers un dépôt git externe. Incluez votre jeton d'authentification dans l'URL (ex. https://user:token@github.com/org/repo.git).",
|
||||
"git_sync_pull_now": "Tirer depuis git",
|
||||
"git_sync_pull_success": "Projet mis à jour depuis le dépôt git avec succès.",
|
||||
"git_sync_pulling": "Récupération…",
|
||||
"git_sync_push_now": "Pousser maintenant",
|
||||
"git_sync_pushing": "Envoi en cours…",
|
||||
"git_sync_push_success": "Projet envoyé vers le dépôt git avec succès.",
|
||||
|
||||
Reference in New Issue
Block a user