move migrations to shared location (#28306)
* fix: correct typedef for Document in helpers.mjs * add move-migrations codemod * update migration paths to use shared migrations directory * move migrations to shared location * fix: update Dockerfile and docker-compose.ci.yml to include migrations directory * feat: add migrations tool to workspaces in package.json * [monorepo] Fix order of docker ignore rules * [web] remove unused docker ignore file * [monorepo] replace old references to migrations folder * [server-ce] copy migrations from new place * [migrations] Inline web scripts Co-authored-by: Brian Gough <brian.gough@overleaf.com> * [migrations] move three web scripts over Co-authored-by: Brian Gough <brian.gough@overleaf.com> * [migrations] add missing collection Co-authored-by: Brian Gough <brian.gough@overleaf.com> * [migrations] remove lodash dependency Co-authored-by: Brian Gough <brian.gough@overleaf.com> * [migrations] avoid mongodb-legacy dependency Co-authored-by: Brian Gough <brian.gough@overleaf.com> * [monorepo] run migrations from tools/migrations Co-authored-by: Brian Gough <brian.gough@overleaf.com> * [migrations] simplify migration for adding gitBridge feature to users * [monorepo] run migrations from tests in all the services * [migrations] add Jenkins pipeline for linting/formatting * [monorepo] fixup running web migrations everywhere * [monorepo] trigger Jenkins builds on changes to mongo migrations * [migrations] add Jenkins pipeline for linting/formatting * [monorepo] build scripts: update devDependencies before deps scanning * [monorepo] build scripts: formerly depend on tools/migrations * [monorepo] run eslint on .mjs files * [migrations] enable more eslint rules and fix all the errors * [rake] fix migrations:list task --------- Co-authored-by: Jakob Ackermann <jakob.ackermann@overleaf.com> GitOrigin-RevId: 14cf69cc1b9405bbc75adbb9a000e555500e0614
This commit is contained in:
committed by
Copybot
co-authored by
Brian Gough
Jakob Ackermann
parent
fbea855690
commit
729e0f5ac9
@@ -1,97 +0,0 @@
|
||||
import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
|
||||
import { promiseMapWithLimit, promisify } from '@overleaf/promise-utils'
|
||||
import { db } from '../app/src/infrastructure/mongodb.js'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import _ from 'lodash'
|
||||
import { scriptRunner } from './lib/ScriptRunner.mjs'
|
||||
|
||||
const sleep = promisify(setTimeout)
|
||||
|
||||
async function main(options, trackProgress) {
|
||||
if (!options) {
|
||||
options = {}
|
||||
}
|
||||
_.defaults(options, {
|
||||
writeConcurrency: parseInt(process.env.WRITE_CONCURRENCY, 10) || 10,
|
||||
performCleanup: process.argv.pop() === '--perform-cleanup',
|
||||
letUserDoubleCheckInputsFor: parseInt(
|
||||
process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR || 10 * 1000,
|
||||
10
|
||||
),
|
||||
})
|
||||
|
||||
await letUserDoubleCheckInputs(options)
|
||||
|
||||
await batchedUpdate(
|
||||
db.projects,
|
||||
// array is not empty ~ array has one item
|
||||
{ 'deletedDocs.0': { $exists: true } },
|
||||
async projects => {
|
||||
await processBatch(projects, options)
|
||||
},
|
||||
{ _id: 1, deletedDocs: 1 },
|
||||
undefined,
|
||||
{ trackProgress }
|
||||
)
|
||||
}
|
||||
|
||||
async function processBatch(projects, options) {
|
||||
await promiseMapWithLimit(
|
||||
options.writeConcurrency,
|
||||
projects,
|
||||
async project => {
|
||||
await processProject(project, options)
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
async function processProject(project, options) {
|
||||
for (const doc of project.deletedDocs) {
|
||||
await backFillDoc(doc)
|
||||
}
|
||||
if (options.performCleanup) {
|
||||
await cleanupProject(project)
|
||||
}
|
||||
}
|
||||
|
||||
async function backFillDoc(doc) {
|
||||
const { name, deletedAt } = doc
|
||||
await db.docs.updateOne({ _id: doc._id }, { $set: { name, deletedAt } })
|
||||
}
|
||||
|
||||
async function cleanupProject(project) {
|
||||
await db.projects.updateOne(
|
||||
{ _id: project._id },
|
||||
{ $set: { deletedDocs: [] } }
|
||||
)
|
||||
}
|
||||
|
||||
async function letUserDoubleCheckInputs(options) {
|
||||
if (options.performCleanup) {
|
||||
console.error('BACK FILLING AND PERFORMING CLEANUP')
|
||||
} else {
|
||||
console.error(
|
||||
'BACK FILLING ONLY - You will need to rerun with --perform-cleanup'
|
||||
)
|
||||
}
|
||||
console.error(
|
||||
'Waiting for you to double check inputs for',
|
||||
options.letUserDoubleCheckInputsFor,
|
||||
'ms'
|
||||
)
|
||||
await sleep(options.letUserDoubleCheckInputsFor)
|
||||
}
|
||||
|
||||
export default main
|
||||
|
||||
if (fileURLToPath(import.meta.url) === process.argv[1]) {
|
||||
try {
|
||||
await scriptRunner(
|
||||
async trackProgress => await main(undefined, trackProgress)
|
||||
)
|
||||
process.exit(0)
|
||||
} catch (error) {
|
||||
console.error({ error })
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
import { db } from '../app/src/infrastructure/mongodb.js'
|
||||
import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const DRY_RUN = !process.argv.includes('--dry-run=false')
|
||||
const LOG_EVERY_IN_S = parseInt(process.env.LOG_EVERY_IN_S, 10) || 5
|
||||
|
||||
async function main(DRY_RUN) {
|
||||
let processed = 0
|
||||
let deleted = 0
|
||||
let lastLog = 0
|
||||
|
||||
function logProgress() {
|
||||
console.log(`rev missing ${processed} | deleted=true ${deleted}`)
|
||||
}
|
||||
|
||||
await batchedUpdate(
|
||||
db.docs,
|
||||
{ rev: { $exists: false } },
|
||||
async docs => {
|
||||
if (!DRY_RUN) {
|
||||
await db.docs.updateMany(
|
||||
{
|
||||
_id: { $in: docs.map(doc => doc._id) },
|
||||
rev: { $exists: false },
|
||||
},
|
||||
{ $set: { rev: 1 } }
|
||||
)
|
||||
}
|
||||
|
||||
processed += docs.length
|
||||
deleted += docs.filter(doc => doc.deleted).length
|
||||
|
||||
if (Date.now() - lastLog >= LOG_EVERY_IN_S * 1000) {
|
||||
logProgress()
|
||||
lastLog = Date.now()
|
||||
}
|
||||
},
|
||||
{
|
||||
_id: 1,
|
||||
deleted: true,
|
||||
}
|
||||
)
|
||||
|
||||
logProgress()
|
||||
}
|
||||
|
||||
export default main
|
||||
|
||||
if (fileURLToPath(import.meta.url) === process.argv[1]) {
|
||||
try {
|
||||
await main(DRY_RUN)
|
||||
console.log('Done.')
|
||||
process.exit(0)
|
||||
} catch (error) {
|
||||
console.error({ error })
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
@@ -1,162 +0,0 @@
|
||||
import { promisify } from 'node:util'
|
||||
import mongodb from 'mongodb-legacy'
|
||||
import {
|
||||
db,
|
||||
READ_PREFERENCE_SECONDARY,
|
||||
} from '../app/src/infrastructure/mongodb.js'
|
||||
import _ from 'lodash'
|
||||
import LRUCache from 'lru-cache'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { scriptRunner } from './lib/ScriptRunner.mjs'
|
||||
|
||||
const { ObjectId } = mongodb
|
||||
const sleep = promisify(setTimeout)
|
||||
|
||||
const NOW_IN_S = Date.now() / 1000
|
||||
const ONE_WEEK_IN_S = 60 * 60 * 24 * 7
|
||||
const TEN_SECONDS = 10 * 1000
|
||||
|
||||
const DUMMY_NAME = 'unknown.tex'
|
||||
const DUMMY_TIME = new Date('2021-04-12T00:00:00.000Z')
|
||||
|
||||
let deletedProjectsCache = null
|
||||
|
||||
function getSecondsFromObjectId(id) {
|
||||
return id.getTimestamp().getTime() / 1000
|
||||
}
|
||||
|
||||
async function main(options) {
|
||||
if (!options) {
|
||||
options = {}
|
||||
}
|
||||
_.defaults(options, {
|
||||
dryRun: process.env.DRY_RUN === 'true',
|
||||
cacheSize: parseInt(process.env.CACHE_SIZE, 10) || 100,
|
||||
firstProjectId: new ObjectId(process.env.FIRST_PROJECT_ID),
|
||||
incrementByS: parseInt(process.env.INCREMENT_BY_S, 10) || ONE_WEEK_IN_S,
|
||||
batchSize: parseInt(process.env.BATCH_SIZE, 10) || 1000,
|
||||
stopAtS: parseInt(process.env.STOP_AT_S, 10) || NOW_IN_S,
|
||||
letUserDoubleCheckInputsFor:
|
||||
parseInt(process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR, 10) || TEN_SECONDS,
|
||||
})
|
||||
|
||||
if (!options.firstProjectId) {
|
||||
console.error('Set FIRST_PROJECT_ID and re-run.')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
deletedProjectsCache = new LRUCache({
|
||||
max: options.cacheSize,
|
||||
})
|
||||
|
||||
await letUserDoubleCheckInputs(options)
|
||||
let startId = options.firstProjectId
|
||||
|
||||
let nProcessed = 0
|
||||
while (getSecondsFromObjectId(startId) <= options.stopAtS) {
|
||||
const end = getSecondsFromObjectId(startId) + options.incrementByS
|
||||
let endId = ObjectId.createFromTime(end)
|
||||
const query = {
|
||||
project_id: {
|
||||
// include edge
|
||||
$gte: startId,
|
||||
// exclude edge
|
||||
$lt: endId,
|
||||
},
|
||||
deleted: true,
|
||||
name: {
|
||||
$exists: false,
|
||||
},
|
||||
}
|
||||
const docs = await db.docs
|
||||
.find(query, { readPreference: READ_PREFERENCE_SECONDARY })
|
||||
.project({ _id: 1, project_id: 1 })
|
||||
.limit(options.batchSize)
|
||||
.toArray()
|
||||
|
||||
if (docs.length) {
|
||||
const docIds = docs.map(doc => doc._id)
|
||||
console.log('Back filling dummy meta data for', JSON.stringify(docIds))
|
||||
await processBatch(docs, options)
|
||||
nProcessed += docIds.length
|
||||
|
||||
if (docs.length === options.batchSize) {
|
||||
endId = docs[docs.length - 1].project_id
|
||||
}
|
||||
}
|
||||
console.error('Processed %d until %s', nProcessed, endId)
|
||||
|
||||
startId = endId
|
||||
}
|
||||
}
|
||||
|
||||
async function getDeletedProject(projectId) {
|
||||
const cacheKey = projectId.toString()
|
||||
if (deletedProjectsCache.has(cacheKey)) {
|
||||
return deletedProjectsCache.get(cacheKey)
|
||||
}
|
||||
const deletedProject = await db.deletedProjects.findOne(
|
||||
{ 'deleterData.deletedProjectId': projectId },
|
||||
{
|
||||
projection: {
|
||||
_id: 1,
|
||||
'project.deletedDocs': 1,
|
||||
},
|
||||
}
|
||||
)
|
||||
deletedProjectsCache.set(cacheKey, deletedProject)
|
||||
return deletedProject
|
||||
}
|
||||
|
||||
async function processBatch(docs, options) {
|
||||
for (const doc of docs) {
|
||||
const { _id: docId, project_id: projectId } = doc
|
||||
const deletedProject = await getDeletedProject(projectId)
|
||||
let name = DUMMY_NAME
|
||||
let deletedAt = DUMMY_TIME
|
||||
if (deletedProject) {
|
||||
const project = deletedProject.project
|
||||
if (project) {
|
||||
const deletedDoc =
|
||||
project.deletedDocs &&
|
||||
project.deletedDocs.find(deletedDoc => docId.equals(deletedDoc._id))
|
||||
if (deletedDoc) {
|
||||
console.log('Found deletedDoc for %s', docId)
|
||||
name = deletedDoc.name
|
||||
deletedAt = deletedDoc.deletedAt
|
||||
} else {
|
||||
console.log('Missing deletedDoc for %s', docId)
|
||||
}
|
||||
} else {
|
||||
console.log('Orphaned deleted doc %s (failed hard deletion)', docId)
|
||||
}
|
||||
} else {
|
||||
console.log('Orphaned deleted doc %s (no deletedProjects entry)', docId)
|
||||
}
|
||||
if (options.dryRun) continue
|
||||
await db.docs.updateOne({ _id: docId }, { $set: { name, deletedAt } })
|
||||
}
|
||||
}
|
||||
|
||||
async function letUserDoubleCheckInputs(options) {
|
||||
console.error('Options:', JSON.stringify(options, null, 2))
|
||||
console.error(
|
||||
'Waiting for you to double check inputs for',
|
||||
options.letUserDoubleCheckInputsFor,
|
||||
'ms'
|
||||
)
|
||||
await sleep(options.letUserDoubleCheckInputsFor)
|
||||
}
|
||||
|
||||
export default main
|
||||
|
||||
if (fileURLToPath(import.meta.url) === process.argv[1]) {
|
||||
try {
|
||||
await scriptRunner(async () => await main())
|
||||
console.error('Done.')
|
||||
process.exit(0)
|
||||
} catch (error) {
|
||||
console.error({ error })
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
import { db } from '../app/src/infrastructure/mongodb.js'
|
||||
import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
|
||||
import minimist from 'minimist'
|
||||
import CollaboratorsInviteHelper from '../app/src/Features/Collaborators/CollaboratorsInviteHelper.js'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const argv = minimist(process.argv.slice(2), {
|
||||
boolean: ['dry-run', 'help'],
|
||||
default: {
|
||||
'dry-run': true,
|
||||
},
|
||||
})
|
||||
|
||||
const DRY_RUN = argv['dry-run']
|
||||
|
||||
async function addTokenHmacField(DRY_RUN) {
|
||||
const query = { tokenHmac: { $exists: false } }
|
||||
|
||||
await batchedUpdate(
|
||||
db.projectInvites,
|
||||
query,
|
||||
async invites => {
|
||||
for (const invite of invites) {
|
||||
console.log(
|
||||
`=> Missing "tokenHmac" token in invitation: ${invite._id.toString()}`
|
||||
)
|
||||
|
||||
if (DRY_RUN) {
|
||||
console.log(
|
||||
`=> DRY RUN - would add "tokenHmac" token to invitation ${invite._id.toString()}`
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
const tokenHmac = CollaboratorsInviteHelper.hashInviteToken(
|
||||
invite.token
|
||||
)
|
||||
|
||||
await db.projectInvites.updateOne(
|
||||
{ _id: invite._id },
|
||||
{ $set: { tokenHmac } }
|
||||
)
|
||||
|
||||
console.log(
|
||||
`=> Added "tokenHmac" token to invitation ${invite._id.toString()}`
|
||||
)
|
||||
}
|
||||
},
|
||||
{ token: 1 }
|
||||
)
|
||||
}
|
||||
|
||||
async function main(DRY_RUN) {
|
||||
await addTokenHmacField(DRY_RUN)
|
||||
}
|
||||
|
||||
export default main
|
||||
|
||||
if (fileURLToPath(import.meta.url) === process.argv[1]) {
|
||||
if (argv.help || argv._.length > 1) {
|
||||
console.error(`Usage: node scripts/backfill_project_invites_token_hmac.mjs
|
||||
Adds a "tokenHmac" field (which is a hashed version of the token) to each project invite record.
|
||||
|
||||
Options:
|
||||
--dry-run finds invitations without HMAC token but does not do any updates
|
||||
`)
|
||||
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
try {
|
||||
await main(DRY_RUN)
|
||||
console.error('Done')
|
||||
process.exit(0)
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import Adapter from '../migrations/lib/adapter.mjs'
|
||||
import Adapter from '../../../tools/migrations/lib/adapter.mjs'
|
||||
import { promises as fs } from 'node:fs'
|
||||
import { join, dirname } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
@@ -1,169 +0,0 @@
|
||||
import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
|
||||
import { promiseMapWithLimit, promisify } from '@overleaf/promise-utils'
|
||||
import { db, ObjectId } from '../app/src/infrastructure/mongodb.js'
|
||||
import _ from 'lodash'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { scriptRunner } from './lib/ScriptRunner.mjs'
|
||||
|
||||
const sleep = promisify(setTimeout)
|
||||
|
||||
async function main(options, trackProgress) {
|
||||
if (!options) {
|
||||
options = {}
|
||||
}
|
||||
_.defaults(options, {
|
||||
dryRun: process.env.DRY_RUN !== 'false',
|
||||
projectId: process.env.PROJECT_ID,
|
||||
userId: process.env.USER_ID,
|
||||
skipUsersMigration: process.env.SKIP_USERS_MIGRATION === 'true',
|
||||
writeConcurrency: parseInt(process.env.WRITE_CONCURRENCY, 10) || 10,
|
||||
letUserDoubleCheckInputsFor: parseInt(
|
||||
process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR || 10 * 1000,
|
||||
10
|
||||
),
|
||||
})
|
||||
|
||||
await letUserDoubleCheckInputs(options)
|
||||
|
||||
if (options.projectId) {
|
||||
console.log('migrating projectId=' + options.projectId)
|
||||
const project = await db.projects.findOne(
|
||||
{ _id: new ObjectId(options.projectId) },
|
||||
{ _id: 1, auditLog: 1 }
|
||||
)
|
||||
if (!project || !project.auditLog) {
|
||||
console.error('unable to process project', project)
|
||||
return
|
||||
}
|
||||
await processProjectsBatch([project], options)
|
||||
} else if (options.userId) {
|
||||
console.log('migrating userId=' + options.userId)
|
||||
const user = await db.users.findOne(
|
||||
{ _id: new ObjectId(options.userId) },
|
||||
{ _id: 1, auditLog: 1 }
|
||||
)
|
||||
if (!user || !user.auditLog) {
|
||||
console.error('unable to process user', user)
|
||||
return
|
||||
}
|
||||
await processUsersBatch([user], options)
|
||||
} else {
|
||||
if (!options.skipUsersMigration) {
|
||||
await batchedUpdate(
|
||||
db.users,
|
||||
{ auditLog: { $exists: true } },
|
||||
async users => {
|
||||
await processUsersBatch(users, options)
|
||||
},
|
||||
{ _id: 1, auditLog: 1 },
|
||||
undefined,
|
||||
{ trackProgress }
|
||||
)
|
||||
}
|
||||
|
||||
// most projects are processed after its owner has been processed, but only those
|
||||
// users with an existing `auditLog` have been taken into consideration, leaving
|
||||
// some projects orphan. This batched update processes all remaining projects.
|
||||
await batchedUpdate(
|
||||
db.projects,
|
||||
{ auditLog: { $exists: true } },
|
||||
async projects => {
|
||||
await processProjectsBatch(projects, options)
|
||||
},
|
||||
{ _id: 1, auditLog: 1 },
|
||||
undefined,
|
||||
{ trackProgress }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async function processUsersBatch(users, options) {
|
||||
if (!users || users.length <= 0) {
|
||||
return
|
||||
}
|
||||
|
||||
const entries = users
|
||||
.map(user => user.auditLog.map(log => ({ ...log, userId: user._id })))
|
||||
.flat()
|
||||
|
||||
if (!options.dryRun && entries?.length > 0) {
|
||||
await db.userAuditLogEntries.insertMany(entries)
|
||||
}
|
||||
|
||||
if (!options.dryRun) {
|
||||
const userIds = users.map(user => user._id)
|
||||
await db.users.updateMany(
|
||||
{ _id: { $in: userIds } },
|
||||
{ $unset: { auditLog: 1 } }
|
||||
)
|
||||
}
|
||||
|
||||
await promiseMapWithLimit(options.writeConcurrency, users, async user => {
|
||||
const projects = await db.projects
|
||||
.find(
|
||||
{ owner_ref: user._id, auditLog: { $exists: true } },
|
||||
{ _id: 1, auditLog: 1 }
|
||||
)
|
||||
.toArray()
|
||||
await processProjectsBatch(projects, options)
|
||||
})
|
||||
}
|
||||
|
||||
async function processProjectsBatch(projects, options) {
|
||||
if (!projects || projects.length <= 0) {
|
||||
return
|
||||
}
|
||||
|
||||
const entries = projects
|
||||
.map(project =>
|
||||
project.auditLog.map(log => ({ ...log, projectId: project._id }))
|
||||
)
|
||||
.flat()
|
||||
|
||||
if (!options.dryRun && entries?.length > 0) {
|
||||
await db.projectAuditLogEntries.insertMany(entries)
|
||||
}
|
||||
|
||||
if (!options.dryRun) {
|
||||
const projectIds = projects.map(project => project._id)
|
||||
await db.projects.updateMany(
|
||||
{ _id: { $in: projectIds } },
|
||||
{ $unset: { auditLog: 1 } }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async function letUserDoubleCheckInputs(options) {
|
||||
const allOptions = {
|
||||
...options,
|
||||
// batchedUpdate() environment variables
|
||||
BATCH_DESCENDING: process.env.BATCH_DESCENDING,
|
||||
BATCH_SIZE: process.env.BATCH_SIZE,
|
||||
VERBOSE_LOGGING: process.env.VERBOSE_LOGGING,
|
||||
BATCH_LAST_ID: process.env.BATCH_LAST_ID,
|
||||
BATCH_RANGE_END: process.env.BATCH_RANGE_END,
|
||||
SKIP_USERS_MIGRATION: process.env.SKIP_USERS_MIGRATION,
|
||||
}
|
||||
console.error('Options:', JSON.stringify(allOptions, null, 2))
|
||||
console.error(
|
||||
'Waiting for you to double check inputs for',
|
||||
options.letUserDoubleCheckInputsFor,
|
||||
'ms'
|
||||
)
|
||||
await sleep(options.letUserDoubleCheckInputsFor)
|
||||
}
|
||||
|
||||
export default main
|
||||
|
||||
if (fileURLToPath(import.meta.url) === process.argv[1]) {
|
||||
try {
|
||||
await scriptRunner(
|
||||
async trackProgress => await main(undefined, trackProgress)
|
||||
)
|
||||
console.log('Done.')
|
||||
process.exit(0)
|
||||
} catch (error) {
|
||||
console.error({ error })
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
@@ -1,221 +0,0 @@
|
||||
import {
|
||||
db,
|
||||
READ_PREFERENCE_SECONDARY,
|
||||
} from '../app/src/infrastructure/mongodb.js'
|
||||
import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
|
||||
import mongodb from 'mongodb-legacy'
|
||||
import minimist from 'minimist'
|
||||
import CollaboratorsHandler from '../app/src/Features/Collaborators/CollaboratorsHandler.js'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const { ObjectId } = mongodb
|
||||
|
||||
const argv = minimist(process.argv.slice(2), {
|
||||
string: ['projects'],
|
||||
boolean: ['dry-run', 'help'],
|
||||
alias: {
|
||||
projects: 'p',
|
||||
},
|
||||
default: {
|
||||
'dry-run': true,
|
||||
},
|
||||
})
|
||||
|
||||
const DRY_RUN = argv['dry-run']
|
||||
const PROJECTS_LIST = argv.projects
|
||||
|
||||
async function findUserIds() {
|
||||
const userIds = new Set()
|
||||
const cursor = db.users.find(
|
||||
{},
|
||||
{
|
||||
projection: { _id: 1 },
|
||||
readPreference: READ_PREFERENCE_SECONDARY,
|
||||
}
|
||||
)
|
||||
for await (const user of cursor) {
|
||||
userIds.add(user._id.toString())
|
||||
if (userIds.size % 1_000_000 === 0) {
|
||||
console.log(`=> ${userIds.size} users added`, new Date().toISOString())
|
||||
}
|
||||
}
|
||||
console.log(`=> User ids count: ${userIds.size}`)
|
||||
return userIds
|
||||
}
|
||||
|
||||
async function fixProjectsWithInvalidTokenAccessRefsIds(
|
||||
DRY_RUN,
|
||||
PROJECTS_LIST
|
||||
) {
|
||||
if (DRY_RUN) {
|
||||
console.log('=> Doing dry run')
|
||||
}
|
||||
|
||||
const DELETED_USER_COLLABORATOR_IDS = new Set()
|
||||
const PROJECTS_WITH_DELETED_USER = new Set()
|
||||
|
||||
// get a set of all users ids as an in-memory cache
|
||||
const userIds = await findUserIds()
|
||||
|
||||
// default query for finding all projects with non-existing/null or non-empty token access fields
|
||||
let query = {
|
||||
$or: [
|
||||
{ tokenAccessReadOnly_refs: { $not: { $type: 'array' } } },
|
||||
{ tokenAccessReadAndWrite_refs: { $not: { $type: 'array' } } },
|
||||
{ 'tokenAccessReadOnly_refs.0': { $exists: true } },
|
||||
{ 'tokenAccessReadAndWrite_refs.0': { $exists: true } },
|
||||
],
|
||||
}
|
||||
|
||||
const projectIds = PROJECTS_LIST?.split(',').map(
|
||||
projectId => new ObjectId(projectId)
|
||||
)
|
||||
|
||||
// query for finding projects passed in as args
|
||||
if (projectIds) {
|
||||
query = { $and: [{ _id: { $in: projectIds } }] }
|
||||
}
|
||||
|
||||
await batchedUpdate(
|
||||
db.projects,
|
||||
query,
|
||||
async projects => {
|
||||
for (const project of projects) {
|
||||
const isTokenAccessFieldMissing =
|
||||
!project.tokenAccessReadOnly_refs ||
|
||||
!project.tokenAccessReadAndWrite_refs
|
||||
project.tokenAccessReadOnly_refs ??= []
|
||||
project.tokenAccessReadAndWrite_refs ??= []
|
||||
|
||||
// update the token access fields if necessary
|
||||
if (isTokenAccessFieldMissing) {
|
||||
if (DRY_RUN) {
|
||||
console.log(
|
||||
`=> DRY RUN - would fix non-existing token access fields in project ${project._id.toString()}`
|
||||
)
|
||||
} else {
|
||||
const fields = [
|
||||
'tokenAccessReadOnly_refs',
|
||||
'tokenAccessReadAndWrite_refs',
|
||||
]
|
||||
for (const field of fields) {
|
||||
await db.projects.updateOne(
|
||||
{
|
||||
_id: project._id,
|
||||
[field]: { $not: { $type: 'array' } },
|
||||
},
|
||||
{ $set: { [field]: [] } }
|
||||
)
|
||||
}
|
||||
console.log(
|
||||
`=> Fixed non-existing token access fields in project ${project._id.toString()}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// find the set of user ids that are in the token access fields
|
||||
// i.e. the set of collaborators
|
||||
const collaboratorIds = new Set()
|
||||
for (const roUserId of project.tokenAccessReadOnly_refs) {
|
||||
collaboratorIds.add(roUserId.toString())
|
||||
}
|
||||
for (const rwUserId of project.tokenAccessReadAndWrite_refs) {
|
||||
collaboratorIds.add(rwUserId.toString())
|
||||
}
|
||||
// determine which collaborator ids are not in the `users` collection
|
||||
// i.e. the user has been deleted
|
||||
const deletedUserIds = new Set()
|
||||
for (const collaboratorId of collaboratorIds) {
|
||||
if (!userIds.has(collaboratorId)) {
|
||||
deletedUserIds.add(collaboratorId)
|
||||
}
|
||||
}
|
||||
|
||||
// double-check that users doesn't exist in the users collection
|
||||
// we don't want to remove users that were added after the initial query
|
||||
const existingUsersCursor = db.users.find(
|
||||
{ _id: { $in: [...deletedUserIds].map(id => new ObjectId(id)) } },
|
||||
{ _id: 1 }
|
||||
)
|
||||
for await (const user of existingUsersCursor) {
|
||||
const id = user._id.toString()
|
||||
deletedUserIds.delete(id)
|
||||
// add the user id to the cache
|
||||
userIds.add(id)
|
||||
}
|
||||
|
||||
// remove the actual deleted users
|
||||
for (const deletedUserId of deletedUserIds) {
|
||||
DELETED_USER_COLLABORATOR_IDS.add(deletedUserId)
|
||||
PROJECTS_WITH_DELETED_USER.add(project._id.toString())
|
||||
console.log(
|
||||
'=> Found deleted user id:',
|
||||
deletedUserId,
|
||||
'in project:',
|
||||
project._id.toString()
|
||||
)
|
||||
if (DRY_RUN) {
|
||||
console.log(
|
||||
`=> DRY RUN - would remove deleted ${deletedUserId} from all projects (found in project ${project._id.toString()})`
|
||||
)
|
||||
continue
|
||||
}
|
||||
console.log(
|
||||
`=> Removing deleted ${deletedUserId} from all projects (found in project ${project._id.toString()})`
|
||||
)
|
||||
await CollaboratorsHandler.promises.removeUserFromAllProjects(
|
||||
new ObjectId(deletedUserId)
|
||||
)
|
||||
}
|
||||
}
|
||||
},
|
||||
{ tokenAccessReadOnly_refs: 1, tokenAccessReadAndWrite_refs: 1 }
|
||||
)
|
||||
|
||||
console.log(
|
||||
`=> ${DRY_RUN ? 'DRY RUN - would delete' : 'Deleted'} user ids (${
|
||||
DELETED_USER_COLLABORATOR_IDS.size
|
||||
})`
|
||||
)
|
||||
if (DELETED_USER_COLLABORATOR_IDS.size) {
|
||||
console.log(Array.from(DELETED_USER_COLLABORATOR_IDS).join('\n'))
|
||||
}
|
||||
console.log(
|
||||
`=> Projects with deleted user ids (${PROJECTS_WITH_DELETED_USER.size})`
|
||||
)
|
||||
if (PROJECTS_WITH_DELETED_USER.size) {
|
||||
console.log(Array.from(PROJECTS_WITH_DELETED_USER).join('\n'))
|
||||
}
|
||||
}
|
||||
|
||||
async function main(DRY_RUN, PROJECTS_LIST) {
|
||||
await fixProjectsWithInvalidTokenAccessRefsIds(DRY_RUN, PROJECTS_LIST)
|
||||
}
|
||||
|
||||
export default main
|
||||
|
||||
if (fileURLToPath(import.meta.url) === process.argv[1]) {
|
||||
if (argv.help || argv._.length > 1) {
|
||||
console.error(`Usage: node scripts/remove_deleted_users_from_token_access_refs.mjs [OPTS]
|
||||
Finds or removes deleted user ids from token access fields
|
||||
"tokenAccessReadOnly_refs" and "tokenAccessReadAndWrite_refs" in the "projects" collection.
|
||||
|
||||
If no projects are specified, all projects will be processed.
|
||||
|
||||
Options:
|
||||
|
||||
--dry-run finds projects and deleted users but does not do any updates
|
||||
--projects list of projects ids to be fixed (comma separated)
|
||||
`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
try {
|
||||
await main(DRY_RUN, PROJECTS_LIST)
|
||||
console.error('Done')
|
||||
process.exit(0)
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user