alois/Verso
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[misc] migrate the app to the native mongo driver

Browse Source 
acceptance tests to follow in a separate commit.
This commit is contained in:
Jakob Ackermann
2020-09-10 17:39:12 +01:00
parent 3a4aa45e2e
commit c630cbb4e3
15 changed files with 135 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/track-changes/app/js/HealthChecker.js
+1 -1
View File
@@ -10,7 +10,7 @@
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const { ObjectId } = require('mongojs')
const { ObjectId } = require('./mongodb')
const request = require('request')
const async = require('async')
const settings = require('settings-sharelatex')
services/track-changes/app/js/MongoAWS.js
+6 -2
View File
@@ -18,7 +18,7 @@ const settings = require('settings-sharelatex')
const logger = require('logger-sharelatex')
const AWS = require('aws-sdk')
const S3S = require('s3-streams')
const { db, ObjectId } = require('./mongojs')
const { db, ObjectId } = require('./mongodb')
const JSONStream = require('JSONStream')
const ReadlineStream = require('byline')
const zlib = require('zlib')
@@ -187,7 +187,11 @@ module.exports = MongoAWS = {
// allow the object to expire, we can always retrieve it again
object.expiresAt = new Date(Date.now() + 7 * DAYS)
logger.log({ project_id, doc_id, pack_id }, 'inserting object from s3')
return db.docHistory.insert(object, callback)
return db.docHistory.insertOne(object, (err, confirmation) => {
if (err) return callback(err)
object._id = confirmation.insertedId
callback(null, object)
})
})
}
}
services/track-changes/app/js/MongoManager.js
+9 -11
View File
@@ -12,7 +12,7 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let MongoManager
const { db, ObjectId } = require('./mongojs')
const { db, ObjectId } = require('./mongodb')
const PackManager = require('./PackManager')
const async = require('async')
const _ = require('underscore')
@@ -25,7 +25,11 @@ module.exports = MongoManager = {
callback = function (error, update) {}
}
return db.docHistory
.find({ doc_id: ObjectId(doc_id.toString()) }, { pack: { $slice: -1 } }) // only return the last entry in a pack
.find(
{ doc_id: ObjectId(doc_id.toString()) },
// only return the last entry in a pack
{ projection: { pack: { $slice: -1 } } }
)
.sort({ v: -1 })
.limit(1)
.toArray(function (error, compressedUpdates) {
@@ -96,7 +100,7 @@ module.exports = MongoManager = {
if (callback == null) {
callback = function (error) {}
}
return db.docHistory.update(
return db.docHistory.updateMany(
{
doc_id: ObjectId(doc_id.toString()),
project_id: { $exists: false }
@@ -104,9 +108,6 @@ module.exports = MongoManager = {
{
$set: { project_id: ObjectId(project_id.toString()) }
},
{
multi: true
},
callback
)
},
@@ -127,7 +128,7 @@ module.exports = MongoManager = {
if (callback == null) {
callback = function (error) {}
}
return db.projectHistoryMetaData.update(
return db.projectHistoryMetaData.updateOne(
{
project_id: ObjectId(project_id)
},
@@ -146,7 +147,7 @@ module.exports = MongoManager = {
if (callback == null) {
callback = function (error) {}
}
return db.docHistory.update(
return db.docHistory.updateMany(
{
project_id: ObjectId(project_id),
temporary: true,
@@ -156,9 +157,6 @@ module.exports = MongoManager = {
$set: { temporary: false },
$unset: { expiresAt: '' }
},
{
multi: true
},
callback
)
},
services/track-changes/app/js/PackManager.js
+26 -24
View File
@@ -18,7 +18,7 @@ const async = require('async')
const _ = require('underscore')
const Bson = require('bson')
const BSON = new Bson()
const { db, ObjectId } = require('./mongojs')
const { db, ObjectId } = require('./mongodb')
const logger = require('logger-sharelatex')
const LockManager = require('./LockManager')
const MongoAWS = require('./MongoAWS')
@@ -220,7 +220,7 @@ module.exports = PackManager = {
{ project_id, doc_id, newUpdates },
'inserting updates into new pack'
)
return db.docHistory.save(newPack, function (err, result) {
return db.docHistory.insertOne(newPack, function (err) {
if (err != null) {
return callback(err)
}
@@ -381,7 +381,10 @@ module.exports = PackManager = {
fetchPacksIfNeeded(project_id, doc_id, pack_ids, callback) {
let id
return db.docHistory
.find({ _id: { $in: pack_ids.map(ObjectId) } }, { _id: 1 })
.find(
{ _id: { $in: pack_ids.map(ObjectId) } },
{ projection: { _id: 1 } }
)
.toArray(function (err, loadedPacks) {
if (err != null) {
return callback(err)
@@ -425,7 +428,10 @@ module.exports = PackManager = {
makeProjectIterator(project_id, before, callback) {
// get all the docHistory Entries
return db.docHistory
.find({ project_id: ObjectId(project_id) }, { pack: false })
.find(
{ project_id: ObjectId(project_id) },
{ projection: { pack: false } }
)
.sort({ 'meta.end_ts': -1 })
.toArray(function (err, packs) {
let pack
@@ -507,7 +513,7 @@ module.exports = PackManager = {
getPackFromIndex(doc_id, pack_id, callback) {
return db.docHistoryIndex.findOne(
{ _id: ObjectId(doc_id.toString()), 'packs._id': pack_id },
{ 'packs.$': 1 },
{ projection: { 'packs.$': 1 } },
callback
)
},
@@ -515,7 +521,7 @@ module.exports = PackManager = {
getLastPackFromIndex(doc_id, callback) {
return db.docHistoryIndex.findOne(
{ _id: ObjectId(doc_id.toString()) },
{ packs: { $slice: -1 } },
{ projection: { packs: { $slice: -1 } } },
function (err, indexPack) {
if (err != null) {
return callback(err)
@@ -602,7 +608,7 @@ module.exports = PackManager = {
expiresAt: { $exists: false }
}
return db.docHistory
.find(query, { pack: false })
.find(query, { projection: { pack: false } })
.sort({ v: 1 })
.toArray(function (err, packs) {
if (err != null) {
@@ -628,7 +634,7 @@ module.exports = PackManager = {
expiresAt: { $exists: false }
}
return db.docHistory
.find(query, { pack: false })
.find(query, { projection: { pack: false } })
.sort({ v: 1 })
.toArray(function (err, packs) {
if (err != null) {
@@ -1069,20 +1075,18 @@ module.exports = PackManager = {
{ project_id, doc_id },
'marking pack as archive in progress status'
)
return db.docHistoryIndex.findAndModify(
return db.docHistoryIndex.findOneAndUpdate(
{
query: {
_id: ObjectId(doc_id.toString()),
packs: { $elemMatch: { _id: pack_id, inS3: { $exists: false } } }
},
fields: { 'packs.$': 1 },
update: { $set: { 'packs.$.inS3': false } }
_id: ObjectId(doc_id.toString()),
packs: { $elemMatch: { _id: pack_id, inS3: { $exists: false } } }
},
{ $set: { 'packs.$.inS3': false } },
{ projection: { 'packs.$': 1 } },
function (err, result) {
if (err != null) {
return callback(err)
}
if (result == null) {
if (!result.value) {
return callback(new Error('archive is already in progress'))
}
logger.log(
@@ -1111,20 +1115,18 @@ module.exports = PackManager = {
markPackAsArchived(project_id, doc_id, pack_id, callback) {
logger.log({ project_id, doc_id, pack_id }, 'marking pack as archived')
return db.docHistoryIndex.findAndModify(
return db.docHistoryIndex.findOneAndUpdate(
{
query: {
_id: ObjectId(doc_id.toString()),
packs: { $elemMatch: { _id: pack_id, inS3: false } }
},
fields: { 'packs.$': 1 },
update: { $set: { 'packs.$.inS3': true } }
_id: ObjectId(doc_id.toString()),
packs: { $elemMatch: { _id: pack_id, inS3: false } }
},
{ $set: { 'packs.$.inS3': true } },
{ projection: { 'packs.$': 1 } },
function (err, result) {
if (err != null) {
return callback(err)
}
if (result == null) {
if (!result.value) {
return callback(new Error('archive is not marked as progress'))
}
logger.log({ project_id, doc_id, pack_id }, 'marked as archived')
services/track-changes/app/js/PackWorker.js
+2 -2
View File
@@ -18,7 +18,7 @@ let project_id, doc_id
const Settings = require('settings-sharelatex')
const async = require('async')
const _ = require('underscore')
const { db, ObjectId } = require('./mongojs')
const { db, ObjectId } = require('./mongodb')
const fs = require('fs')
const Metrics = require('metrics-sharelatex')
Metrics.initialize('track-changes')
@@ -174,7 +174,7 @@ if (pending != null) {
_id: { $lt: ObjectIdFromDate(oneWeekAgo) },
last_checked: { $lt: oneWeekAgo }
},
{ _id: 1, doc_id: 1, project_id: 1 }
{ projection: { _id: 1, doc_id: 1, project_id: 1 } }
)
.sort({
last_checked: 1
services/track-changes/app/js/mongodb.js
+30
View File
@@ -0,0 +1,30 @@
const Settings = require('settings-sharelatex')
const { MongoClient, ObjectId } = require('mongodb')
const clientPromise = MongoClient.connect(
Settings.mongo.url,
Settings.mongo.options
)
let setupDbPromise
async function waitForDb() {
if (!setupDbPromise) {
setupDbPromise = setupDb()
}
await setupDbPromise
}
const db = {}
async function setupDb() {
const internalDb = (await clientPromise).db()
db.docHistory = internalDb.collection('docHistory')
db.docHistoryIndex = internalDb.collection('docHistoryIndex')
db.projectHistoryMetaData = internalDb.collection('projectHistoryMetaData')
}
module.exports = {
db,
ObjectId,
waitForDb
}