move migrations to shared location (#28306)

* fix: correct typedef for Document in helpers.mjs

* add move-migrations codemod

* update migration paths to use shared migrations directory

* move migrations to shared location

* fix: update Dockerfile and docker-compose.ci.yml to include migrations directory

* feat: add migrations tool to workspaces in package.json

* [monorepo] Fix order of docker ignore rules

* [web] remove unused docker ignore file

* [monorepo] replace old references to migrations folder

* [server-ce] copy migrations from new place

* [migrations] Inline web scripts

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] move three web scripts over

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] add missing collection

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] remove lodash dependency

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] avoid mongodb-legacy dependency

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [monorepo] run migrations from tools/migrations

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] simplify migration for adding gitBridge feature to users

* [monorepo] run migrations from tests in all the services

* [migrations] add Jenkins pipeline for linting/formatting

* [monorepo] fixup running web migrations everywhere

* [monorepo] trigger Jenkins builds on changes to mongo migrations

* [migrations] add Jenkins pipeline for linting/formatting

* [monorepo] build scripts: update devDependencies before deps scanning

* [monorepo] build scripts: formerly depend on tools/migrations

* [monorepo] run eslint on .mjs files

* [migrations] enable more eslint rules and fix all the errors

* [rake] fix migrations:list task

---------

Co-authored-by: Jakob Ackermann <jakob.ackermann@overleaf.com>
GitOrigin-RevId: 14cf69cc1b9405bbc75adbb9a000e555500e0614
This commit is contained in:
Brian Gough
2025-10-16 08:07:37 +00:00
committed by Copybot
co-authored by Brian Gough Jakob Ackermann
parent fbea855690
commit 729e0f5ac9
249 changed files with 802 additions and 984 deletions
@@ -1,97 +0,0 @@
import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
import { promiseMapWithLimit, promisify } from '@overleaf/promise-utils'
import { db } from '../app/src/infrastructure/mongodb.js'
import { fileURLToPath } from 'node:url'
import _ from 'lodash'
import { scriptRunner } from './lib/ScriptRunner.mjs'
const sleep = promisify(setTimeout)
async function main(options, trackProgress) {
if (!options) {
options = {}
}
_.defaults(options, {
writeConcurrency: parseInt(process.env.WRITE_CONCURRENCY, 10) || 10,
performCleanup: process.argv.pop() === '--perform-cleanup',
letUserDoubleCheckInputsFor: parseInt(
process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR || 10 * 1000,
10
),
})
await letUserDoubleCheckInputs(options)
await batchedUpdate(
db.projects,
// array is not empty ~ array has one item
{ 'deletedDocs.0': { $exists: true } },
async projects => {
await processBatch(projects, options)
},
{ _id: 1, deletedDocs: 1 },
undefined,
{ trackProgress }
)
}
async function processBatch(projects, options) {
await promiseMapWithLimit(
options.writeConcurrency,
projects,
async project => {
await processProject(project, options)
}
)
}
async function processProject(project, options) {
for (const doc of project.deletedDocs) {
await backFillDoc(doc)
}
if (options.performCleanup) {
await cleanupProject(project)
}
}
async function backFillDoc(doc) {
const { name, deletedAt } = doc
await db.docs.updateOne({ _id: doc._id }, { $set: { name, deletedAt } })
}
async function cleanupProject(project) {
await db.projects.updateOne(
{ _id: project._id },
{ $set: { deletedDocs: [] } }
)
}
async function letUserDoubleCheckInputs(options) {
if (options.performCleanup) {
console.error('BACK FILLING AND PERFORMING CLEANUP')
} else {
console.error(
'BACK FILLING ONLY - You will need to rerun with --perform-cleanup'
)
}
console.error(
'Waiting for you to double check inputs for',
options.letUserDoubleCheckInputsFor,
'ms'
)
await sleep(options.letUserDoubleCheckInputsFor)
}
export default main
if (fileURLToPath(import.meta.url) === process.argv[1]) {
try {
await scriptRunner(
async trackProgress => await main(undefined, trackProgress)
)
process.exit(0)
} catch (error) {
console.error({ error })
process.exit(1)
}
}
@@ -1,59 +0,0 @@
import { db } from '../app/src/infrastructure/mongodb.js'
import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
import { fileURLToPath } from 'node:url'
const DRY_RUN = !process.argv.includes('--dry-run=false')
const LOG_EVERY_IN_S = parseInt(process.env.LOG_EVERY_IN_S, 10) || 5
async function main(DRY_RUN) {
let processed = 0
let deleted = 0
let lastLog = 0
function logProgress() {
console.log(`rev missing ${processed} | deleted=true ${deleted}`)
}
await batchedUpdate(
db.docs,
{ rev: { $exists: false } },
async docs => {
if (!DRY_RUN) {
await db.docs.updateMany(
{
_id: { $in: docs.map(doc => doc._id) },
rev: { $exists: false },
},
{ $set: { rev: 1 } }
)
}
processed += docs.length
deleted += docs.filter(doc => doc.deleted).length
if (Date.now() - lastLog >= LOG_EVERY_IN_S * 1000) {
logProgress()
lastLog = Date.now()
}
},
{
_id: 1,
deleted: true,
}
)
logProgress()
}
export default main
if (fileURLToPath(import.meta.url) === process.argv[1]) {
try {
await main(DRY_RUN)
console.log('Done.')
process.exit(0)
} catch (error) {
console.error({ error })
process.exit(1)
}
}
@@ -1,162 +0,0 @@
import { promisify } from 'node:util'
import mongodb from 'mongodb-legacy'
import {
db,
READ_PREFERENCE_SECONDARY,
} from '../app/src/infrastructure/mongodb.js'
import _ from 'lodash'
import LRUCache from 'lru-cache'
import { fileURLToPath } from 'node:url'
import { scriptRunner } from './lib/ScriptRunner.mjs'
const { ObjectId } = mongodb
const sleep = promisify(setTimeout)
const NOW_IN_S = Date.now() / 1000
const ONE_WEEK_IN_S = 60 * 60 * 24 * 7
const TEN_SECONDS = 10 * 1000
const DUMMY_NAME = 'unknown.tex'
const DUMMY_TIME = new Date('2021-04-12T00:00:00.000Z')
let deletedProjectsCache = null
function getSecondsFromObjectId(id) {
return id.getTimestamp().getTime() / 1000
}
async function main(options) {
if (!options) {
options = {}
}
_.defaults(options, {
dryRun: process.env.DRY_RUN === 'true',
cacheSize: parseInt(process.env.CACHE_SIZE, 10) || 100,
firstProjectId: new ObjectId(process.env.FIRST_PROJECT_ID),
incrementByS: parseInt(process.env.INCREMENT_BY_S, 10) || ONE_WEEK_IN_S,
batchSize: parseInt(process.env.BATCH_SIZE, 10) || 1000,
stopAtS: parseInt(process.env.STOP_AT_S, 10) || NOW_IN_S,
letUserDoubleCheckInputsFor:
parseInt(process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR, 10) || TEN_SECONDS,
})
if (!options.firstProjectId) {
console.error('Set FIRST_PROJECT_ID and re-run.')
process.exit(1)
}
deletedProjectsCache = new LRUCache({
max: options.cacheSize,
})
await letUserDoubleCheckInputs(options)
let startId = options.firstProjectId
let nProcessed = 0
while (getSecondsFromObjectId(startId) <= options.stopAtS) {
const end = getSecondsFromObjectId(startId) + options.incrementByS
let endId = ObjectId.createFromTime(end)
const query = {
project_id: {
// include edge
$gte: startId,
// exclude edge
$lt: endId,
},
deleted: true,
name: {
$exists: false,
},
}
const docs = await db.docs
.find(query, { readPreference: READ_PREFERENCE_SECONDARY })
.project({ _id: 1, project_id: 1 })
.limit(options.batchSize)
.toArray()
if (docs.length) {
const docIds = docs.map(doc => doc._id)
console.log('Back filling dummy meta data for', JSON.stringify(docIds))
await processBatch(docs, options)
nProcessed += docIds.length
if (docs.length === options.batchSize) {
endId = docs[docs.length - 1].project_id
}
}
console.error('Processed %d until %s', nProcessed, endId)
startId = endId
}
}
async function getDeletedProject(projectId) {
const cacheKey = projectId.toString()
if (deletedProjectsCache.has(cacheKey)) {
return deletedProjectsCache.get(cacheKey)
}
const deletedProject = await db.deletedProjects.findOne(
{ 'deleterData.deletedProjectId': projectId },
{
projection: {
_id: 1,
'project.deletedDocs': 1,
},
}
)
deletedProjectsCache.set(cacheKey, deletedProject)
return deletedProject
}
async function processBatch(docs, options) {
for (const doc of docs) {
const { _id: docId, project_id: projectId } = doc
const deletedProject = await getDeletedProject(projectId)
let name = DUMMY_NAME
let deletedAt = DUMMY_TIME
if (deletedProject) {
const project = deletedProject.project
if (project) {
const deletedDoc =
project.deletedDocs &&
project.deletedDocs.find(deletedDoc => docId.equals(deletedDoc._id))
if (deletedDoc) {
console.log('Found deletedDoc for %s', docId)
name = deletedDoc.name
deletedAt = deletedDoc.deletedAt
} else {
console.log('Missing deletedDoc for %s', docId)
}
} else {
console.log('Orphaned deleted doc %s (failed hard deletion)', docId)
}
} else {
console.log('Orphaned deleted doc %s (no deletedProjects entry)', docId)
}
if (options.dryRun) continue
await db.docs.updateOne({ _id: docId }, { $set: { name, deletedAt } })
}
}
async function letUserDoubleCheckInputs(options) {
console.error('Options:', JSON.stringify(options, null, 2))
console.error(
'Waiting for you to double check inputs for',
options.letUserDoubleCheckInputsFor,
'ms'
)
await sleep(options.letUserDoubleCheckInputsFor)
}
export default main
if (fileURLToPath(import.meta.url) === process.argv[1]) {
try {
await scriptRunner(async () => await main())
console.error('Done.')
process.exit(0)
} catch (error) {
console.error({ error })
process.exit(1)
}
}
@@ -1,79 +0,0 @@
import { db } from '../app/src/infrastructure/mongodb.js'
import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
import minimist from 'minimist'
import CollaboratorsInviteHelper from '../app/src/Features/Collaborators/CollaboratorsInviteHelper.js'
import { fileURLToPath } from 'node:url'
const argv = minimist(process.argv.slice(2), {
boolean: ['dry-run', 'help'],
default: {
'dry-run': true,
},
})
const DRY_RUN = argv['dry-run']
async function addTokenHmacField(DRY_RUN) {
const query = { tokenHmac: { $exists: false } }
await batchedUpdate(
db.projectInvites,
query,
async invites => {
for (const invite of invites) {
console.log(
`=> Missing "tokenHmac" token in invitation: ${invite._id.toString()}`
)
if (DRY_RUN) {
console.log(
`=> DRY RUN - would add "tokenHmac" token to invitation ${invite._id.toString()}`
)
continue
}
const tokenHmac = CollaboratorsInviteHelper.hashInviteToken(
invite.token
)
await db.projectInvites.updateOne(
{ _id: invite._id },
{ $set: { tokenHmac } }
)
console.log(
`=> Added "tokenHmac" token to invitation ${invite._id.toString()}`
)
}
},
{ token: 1 }
)
}
async function main(DRY_RUN) {
await addTokenHmacField(DRY_RUN)
}
export default main
if (fileURLToPath(import.meta.url) === process.argv[1]) {
if (argv.help || argv._.length > 1) {
console.error(`Usage: node scripts/backfill_project_invites_token_hmac.mjs
Adds a "tokenHmac" field (which is a hashed version of the token) to each project invite record.
Options:
--dry-run finds invitations without HMAC token but does not do any updates
`)
process.exit(1)
}
try {
await main(DRY_RUN)
console.error('Done')
process.exit(0)
} catch (error) {
console.error(error)
process.exit(1)
}
}
+1 -1
View File
@@ -1,4 +1,4 @@
import Adapter from '../migrations/lib/adapter.mjs'
import Adapter from '../../../tools/migrations/lib/adapter.mjs'
import { promises as fs } from 'node:fs'
import { join, dirname } from 'node:path'
import { fileURLToPath } from 'node:url'
-169
View File
@@ -1,169 +0,0 @@
import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
import { promiseMapWithLimit, promisify } from '@overleaf/promise-utils'
import { db, ObjectId } from '../app/src/infrastructure/mongodb.js'
import _ from 'lodash'
import { fileURLToPath } from 'node:url'
import { scriptRunner } from './lib/ScriptRunner.mjs'
const sleep = promisify(setTimeout)
async function main(options, trackProgress) {
if (!options) {
options = {}
}
_.defaults(options, {
dryRun: process.env.DRY_RUN !== 'false',
projectId: process.env.PROJECT_ID,
userId: process.env.USER_ID,
skipUsersMigration: process.env.SKIP_USERS_MIGRATION === 'true',
writeConcurrency: parseInt(process.env.WRITE_CONCURRENCY, 10) || 10,
letUserDoubleCheckInputsFor: parseInt(
process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR || 10 * 1000,
10
),
})
await letUserDoubleCheckInputs(options)
if (options.projectId) {
console.log('migrating projectId=' + options.projectId)
const project = await db.projects.findOne(
{ _id: new ObjectId(options.projectId) },
{ _id: 1, auditLog: 1 }
)
if (!project || !project.auditLog) {
console.error('unable to process project', project)
return
}
await processProjectsBatch([project], options)
} else if (options.userId) {
console.log('migrating userId=' + options.userId)
const user = await db.users.findOne(
{ _id: new ObjectId(options.userId) },
{ _id: 1, auditLog: 1 }
)
if (!user || !user.auditLog) {
console.error('unable to process user', user)
return
}
await processUsersBatch([user], options)
} else {
if (!options.skipUsersMigration) {
await batchedUpdate(
db.users,
{ auditLog: { $exists: true } },
async users => {
await processUsersBatch(users, options)
},
{ _id: 1, auditLog: 1 },
undefined,
{ trackProgress }
)
}
// most projects are processed after its owner has been processed, but only those
// users with an existing `auditLog` have been taken into consideration, leaving
// some projects orphan. This batched update processes all remaining projects.
await batchedUpdate(
db.projects,
{ auditLog: { $exists: true } },
async projects => {
await processProjectsBatch(projects, options)
},
{ _id: 1, auditLog: 1 },
undefined,
{ trackProgress }
)
}
}
async function processUsersBatch(users, options) {
if (!users || users.length <= 0) {
return
}
const entries = users
.map(user => user.auditLog.map(log => ({ ...log, userId: user._id })))
.flat()
if (!options.dryRun && entries?.length > 0) {
await db.userAuditLogEntries.insertMany(entries)
}
if (!options.dryRun) {
const userIds = users.map(user => user._id)
await db.users.updateMany(
{ _id: { $in: userIds } },
{ $unset: { auditLog: 1 } }
)
}
await promiseMapWithLimit(options.writeConcurrency, users, async user => {
const projects = await db.projects
.find(
{ owner_ref: user._id, auditLog: { $exists: true } },
{ _id: 1, auditLog: 1 }
)
.toArray()
await processProjectsBatch(projects, options)
})
}
async function processProjectsBatch(projects, options) {
if (!projects || projects.length <= 0) {
return
}
const entries = projects
.map(project =>
project.auditLog.map(log => ({ ...log, projectId: project._id }))
)
.flat()
if (!options.dryRun && entries?.length > 0) {
await db.projectAuditLogEntries.insertMany(entries)
}
if (!options.dryRun) {
const projectIds = projects.map(project => project._id)
await db.projects.updateMany(
{ _id: { $in: projectIds } },
{ $unset: { auditLog: 1 } }
)
}
}
async function letUserDoubleCheckInputs(options) {
const allOptions = {
...options,
// batchedUpdate() environment variables
BATCH_DESCENDING: process.env.BATCH_DESCENDING,
BATCH_SIZE: process.env.BATCH_SIZE,
VERBOSE_LOGGING: process.env.VERBOSE_LOGGING,
BATCH_LAST_ID: process.env.BATCH_LAST_ID,
BATCH_RANGE_END: process.env.BATCH_RANGE_END,
SKIP_USERS_MIGRATION: process.env.SKIP_USERS_MIGRATION,
}
console.error('Options:', JSON.stringify(allOptions, null, 2))
console.error(
'Waiting for you to double check inputs for',
options.letUserDoubleCheckInputsFor,
'ms'
)
await sleep(options.letUserDoubleCheckInputsFor)
}
export default main
if (fileURLToPath(import.meta.url) === process.argv[1]) {
try {
await scriptRunner(
async trackProgress => await main(undefined, trackProgress)
)
console.log('Done.')
process.exit(0)
} catch (error) {
console.error({ error })
process.exit(1)
}
}
@@ -1,221 +0,0 @@
import {
db,
READ_PREFERENCE_SECONDARY,
} from '../app/src/infrastructure/mongodb.js'
import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
import mongodb from 'mongodb-legacy'
import minimist from 'minimist'
import CollaboratorsHandler from '../app/src/Features/Collaborators/CollaboratorsHandler.js'
import { fileURLToPath } from 'node:url'
const { ObjectId } = mongodb
const argv = minimist(process.argv.slice(2), {
string: ['projects'],
boolean: ['dry-run', 'help'],
alias: {
projects: 'p',
},
default: {
'dry-run': true,
},
})
const DRY_RUN = argv['dry-run']
const PROJECTS_LIST = argv.projects
async function findUserIds() {
const userIds = new Set()
const cursor = db.users.find(
{},
{
projection: { _id: 1 },
readPreference: READ_PREFERENCE_SECONDARY,
}
)
for await (const user of cursor) {
userIds.add(user._id.toString())
if (userIds.size % 1_000_000 === 0) {
console.log(`=> ${userIds.size} users added`, new Date().toISOString())
}
}
console.log(`=> User ids count: ${userIds.size}`)
return userIds
}
async function fixProjectsWithInvalidTokenAccessRefsIds(
DRY_RUN,
PROJECTS_LIST
) {
if (DRY_RUN) {
console.log('=> Doing dry run')
}
const DELETED_USER_COLLABORATOR_IDS = new Set()
const PROJECTS_WITH_DELETED_USER = new Set()
// get a set of all users ids as an in-memory cache
const userIds = await findUserIds()
// default query for finding all projects with non-existing/null or non-empty token access fields
let query = {
$or: [
{ tokenAccessReadOnly_refs: { $not: { $type: 'array' } } },
{ tokenAccessReadAndWrite_refs: { $not: { $type: 'array' } } },
{ 'tokenAccessReadOnly_refs.0': { $exists: true } },
{ 'tokenAccessReadAndWrite_refs.0': { $exists: true } },
],
}
const projectIds = PROJECTS_LIST?.split(',').map(
projectId => new ObjectId(projectId)
)
// query for finding projects passed in as args
if (projectIds) {
query = { $and: [{ _id: { $in: projectIds } }] }
}
await batchedUpdate(
db.projects,
query,
async projects => {
for (const project of projects) {
const isTokenAccessFieldMissing =
!project.tokenAccessReadOnly_refs ||
!project.tokenAccessReadAndWrite_refs
project.tokenAccessReadOnly_refs ??= []
project.tokenAccessReadAndWrite_refs ??= []
// update the token access fields if necessary
if (isTokenAccessFieldMissing) {
if (DRY_RUN) {
console.log(
`=> DRY RUN - would fix non-existing token access fields in project ${project._id.toString()}`
)
} else {
const fields = [
'tokenAccessReadOnly_refs',
'tokenAccessReadAndWrite_refs',
]
for (const field of fields) {
await db.projects.updateOne(
{
_id: project._id,
[field]: { $not: { $type: 'array' } },
},
{ $set: { [field]: [] } }
)
}
console.log(
`=> Fixed non-existing token access fields in project ${project._id.toString()}`
)
}
}
// find the set of user ids that are in the token access fields
// i.e. the set of collaborators
const collaboratorIds = new Set()
for (const roUserId of project.tokenAccessReadOnly_refs) {
collaboratorIds.add(roUserId.toString())
}
for (const rwUserId of project.tokenAccessReadAndWrite_refs) {
collaboratorIds.add(rwUserId.toString())
}
// determine which collaborator ids are not in the `users` collection
// i.e. the user has been deleted
const deletedUserIds = new Set()
for (const collaboratorId of collaboratorIds) {
if (!userIds.has(collaboratorId)) {
deletedUserIds.add(collaboratorId)
}
}
// double-check that users doesn't exist in the users collection
// we don't want to remove users that were added after the initial query
const existingUsersCursor = db.users.find(
{ _id: { $in: [...deletedUserIds].map(id => new ObjectId(id)) } },
{ _id: 1 }
)
for await (const user of existingUsersCursor) {
const id = user._id.toString()
deletedUserIds.delete(id)
// add the user id to the cache
userIds.add(id)
}
// remove the actual deleted users
for (const deletedUserId of deletedUserIds) {
DELETED_USER_COLLABORATOR_IDS.add(deletedUserId)
PROJECTS_WITH_DELETED_USER.add(project._id.toString())
console.log(
'=> Found deleted user id:',
deletedUserId,
'in project:',
project._id.toString()
)
if (DRY_RUN) {
console.log(
`=> DRY RUN - would remove deleted ${deletedUserId} from all projects (found in project ${project._id.toString()})`
)
continue
}
console.log(
`=> Removing deleted ${deletedUserId} from all projects (found in project ${project._id.toString()})`
)
await CollaboratorsHandler.promises.removeUserFromAllProjects(
new ObjectId(deletedUserId)
)
}
}
},
{ tokenAccessReadOnly_refs: 1, tokenAccessReadAndWrite_refs: 1 }
)
console.log(
`=> ${DRY_RUN ? 'DRY RUN - would delete' : 'Deleted'} user ids (${
DELETED_USER_COLLABORATOR_IDS.size
})`
)
if (DELETED_USER_COLLABORATOR_IDS.size) {
console.log(Array.from(DELETED_USER_COLLABORATOR_IDS).join('\n'))
}
console.log(
`=> Projects with deleted user ids (${PROJECTS_WITH_DELETED_USER.size})`
)
if (PROJECTS_WITH_DELETED_USER.size) {
console.log(Array.from(PROJECTS_WITH_DELETED_USER).join('\n'))
}
}
async function main(DRY_RUN, PROJECTS_LIST) {
await fixProjectsWithInvalidTokenAccessRefsIds(DRY_RUN, PROJECTS_LIST)
}
export default main
if (fileURLToPath(import.meta.url) === process.argv[1]) {
if (argv.help || argv._.length > 1) {
console.error(`Usage: node scripts/remove_deleted_users_from_token_access_refs.mjs [OPTS]
Finds or removes deleted user ids from token access fields
"tokenAccessReadOnly_refs" and "tokenAccessReadAndWrite_refs" in the "projects" collection.
If no projects are specified, all projects will be processed.
Options:
--dry-run finds projects and deleted users but does not do any updates
--projects list of projects ids to be fixed (comma separated)
`)
process.exit(1)
}
try {
await main(DRY_RUN, PROJECTS_LIST)
console.error('Done')
process.exit(0)
} catch (error) {
console.error(error)
process.exit(1)
}
}