alois/Verso
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[misc] migrate the app to the native mongo driver

Browse Source 
acceptance tests to follow in a separate commit.
This commit is contained in:
Jakob Ackermann
2020-09-10 12:58:06 +01:00
parent 3a4aa45e2e
commit c630cbb4e3
15 changed files with 135 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/track-changes/app.js
+21 -12
View File
@@ -44,6 +44,7 @@ Metrics.memory.monitor(logger)
const childProcess = require('child_process')
const mongodb = require('./app/js/mongodb')
const HttpController = require('./app/js/HttpController')
const express = require('express')
const bodyParser = require('body-parser')
@@ -128,18 +129,26 @@ const host =
if (!module.parent) {
// Called directly
app.listen(port, host, function (error) {
if (error != null) {
return logger.error(
{ err: error },
'could not start track-changes server'
)
} else {
return logger.info(
`trackchanges starting up, listening on ${host}:${port}`
)
}
})
mongodb
.waitForDb()
.then(() => {
app.listen(port, host, function (error) {
if (error != null) {
return logger.error(
{ err: error },
'could not start track-changes server'
)
} else {
return logger.info(
`trackchanges starting up, listening on ${host}:${port}`
)
}
})
})
.catch((err) => {
logger.fatal({ err }, 'Cannot connect to mongo. Exiting.')
process.exit(1)
})
}
module.exports = app
services/track-changes/app/js/HealthChecker.js
+1 -1
View File
@@ -10,7 +10,7 @@
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const { ObjectId } = require('mongojs')
const { ObjectId } = require('./mongodb')
const request = require('request')
const async = require('async')
const settings = require('settings-sharelatex')
services/track-changes/app/js/MongoAWS.js
+6 -2
View File
@@ -18,7 +18,7 @@ const settings = require('settings-sharelatex')
const logger = require('logger-sharelatex')
const AWS = require('aws-sdk')
const S3S = require('s3-streams')
const { db, ObjectId } = require('./mongojs')
const { db, ObjectId } = require('./mongodb')
const JSONStream = require('JSONStream')
const ReadlineStream = require('byline')
const zlib = require('zlib')
@@ -187,7 +187,11 @@ module.exports = MongoAWS = {
// allow the object to expire, we can always retrieve it again
object.expiresAt = new Date(Date.now() + 7 * DAYS)
logger.log({ project_id, doc_id, pack_id }, 'inserting object from s3')
return db.docHistory.insert(object, callback)
return db.docHistory.insertOne(object, (err, confirmation) => {
if (err) return callback(err)
object._id = confirmation.insertedId
callback(null, object)
})
})
}
}
services/track-changes/app/js/MongoManager.js
+9 -11
View File
@@ -12,7 +12,7 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let MongoManager
const { db, ObjectId } = require('./mongojs')
const { db, ObjectId } = require('./mongodb')
const PackManager = require('./PackManager')
const async = require('async')
const _ = require('underscore')
@@ -25,7 +25,11 @@ module.exports = MongoManager = {
callback = function (error, update) {}
}
return db.docHistory
.find({ doc_id: ObjectId(doc_id.toString()) }, { pack: { $slice: -1 } }) // only return the last entry in a pack
.find(
{ doc_id: ObjectId(doc_id.toString()) },
// only return the last entry in a pack
{ projection: { pack: { $slice: -1 } } }
)
.sort({ v: -1 })
.limit(1)
.toArray(function (error, compressedUpdates) {
@@ -96,7 +100,7 @@ module.exports = MongoManager = {
if (callback == null) {
callback = function (error) {}
}
return db.docHistory.update(
return db.docHistory.updateMany(
{
doc_id: ObjectId(doc_id.toString()),
project_id: { $exists: false }
@@ -104,9 +108,6 @@ module.exports = MongoManager = {
{
$set: { project_id: ObjectId(project_id.toString()) }
},
{
multi: true
},
callback
)
},
@@ -127,7 +128,7 @@ module.exports = MongoManager = {
if (callback == null) {
callback = function (error) {}
}
return db.projectHistoryMetaData.update(
return db.projectHistoryMetaData.updateOne(
{
project_id: ObjectId(project_id)
},
@@ -146,7 +147,7 @@ module.exports = MongoManager = {
if (callback == null) {
callback = function (error) {}
}
return db.docHistory.update(
return db.docHistory.updateMany(
{
project_id: ObjectId(project_id),
temporary: true,
@@ -156,9 +157,6 @@ module.exports = MongoManager = {
$set: { temporary: false },
$unset: { expiresAt: '' }
},
{
multi: true
},
callback
)
},
services/track-changes/app/js/PackManager.js
+26 -24
View File
@@ -18,7 +18,7 @@ const async = require('async')
const _ = require('underscore')
const Bson = require('bson')
const BSON = new Bson()
const { db, ObjectId } = require('./mongojs')
const { db, ObjectId } = require('./mongodb')
const logger = require('logger-sharelatex')
const LockManager = require('./LockManager')
const MongoAWS = require('./MongoAWS')
@@ -220,7 +220,7 @@ module.exports = PackManager = {
{ project_id, doc_id, newUpdates },
'inserting updates into new pack'
)
return db.docHistory.save(newPack, function (err, result) {
return db.docHistory.insertOne(newPack, function (err) {
if (err != null) {
return callback(err)
}
@@ -381,7 +381,10 @@ module.exports = PackManager = {
fetchPacksIfNeeded(project_id, doc_id, pack_ids, callback) {
let id
return db.docHistory
.find({ _id: { $in: pack_ids.map(ObjectId) } }, { _id: 1 })
.find(
{ _id: { $in: pack_ids.map(ObjectId) } },
{ projection: { _id: 1 } }
)
.toArray(function (err, loadedPacks) {
if (err != null) {
return callback(err)
@@ -425,7 +428,10 @@ module.exports = PackManager = {
makeProjectIterator(project_id, before, callback) {
// get all the docHistory Entries
return db.docHistory
.find({ project_id: ObjectId(project_id) }, { pack: false })
.find(
{ project_id: ObjectId(project_id) },
{ projection: { pack: false } }
)
.sort({ 'meta.end_ts': -1 })
.toArray(function (err, packs) {
let pack
@@ -507,7 +513,7 @@ module.exports = PackManager = {
getPackFromIndex(doc_id, pack_id, callback) {
return db.docHistoryIndex.findOne(
{ _id: ObjectId(doc_id.toString()), 'packs._id': pack_id },
{ 'packs.$': 1 },
{ projection: { 'packs.$': 1 } },
callback
)
},
@@ -515,7 +521,7 @@ module.exports = PackManager = {
getLastPackFromIndex(doc_id, callback) {
return db.docHistoryIndex.findOne(
{ _id: ObjectId(doc_id.toString()) },
{ packs: { $slice: -1 } },
{ projection: { packs: { $slice: -1 } } },
function (err, indexPack) {
if (err != null) {
return callback(err)
@@ -602,7 +608,7 @@ module.exports = PackManager = {
expiresAt: { $exists: false }
}
return db.docHistory
.find(query, { pack: false })
.find(query, { projection: { pack: false } })
.sort({ v: 1 })
.toArray(function (err, packs) {
if (err != null) {
@@ -628,7 +634,7 @@ module.exports = PackManager = {
expiresAt: { $exists: false }
}
return db.docHistory
.find(query, { pack: false })
.find(query, { projection: { pack: false } })
.sort({ v: 1 })
.toArray(function (err, packs) {
if (err != null) {
@@ -1069,20 +1075,18 @@ module.exports = PackManager = {
{ project_id, doc_id },
'marking pack as archive in progress status'
)
return db.docHistoryIndex.findAndModify(
return db.docHistoryIndex.findOneAndUpdate(
{
query: {
_id: ObjectId(doc_id.toString()),
packs: { $elemMatch: { _id: pack_id, inS3: { $exists: false } } }
},
fields: { 'packs.$': 1 },
update: { $set: { 'packs.$.inS3': false } }
_id: ObjectId(doc_id.toString()),
packs: { $elemMatch: { _id: pack_id, inS3: { $exists: false } } }
},
{ $set: { 'packs.$.inS3': false } },
{ projection: { 'packs.$': 1 } },
function (err, result) {
if (err != null) {
return callback(err)
}
if (result == null) {
if (!result.value) {
return callback(new Error('archive is already in progress'))
}
logger.log(
@@ -1111,20 +1115,18 @@ module.exports = PackManager = {
markPackAsArchived(project_id, doc_id, pack_id, callback) {
logger.log({ project_id, doc_id, pack_id }, 'marking pack as archived')
return db.docHistoryIndex.findAndModify(
return db.docHistoryIndex.findOneAndUpdate(
{
query: {
_id: ObjectId(doc_id.toString()),
packs: { $elemMatch: { _id: pack_id, inS3: false } }
},
fields: { 'packs.$': 1 },
update: { $set: { 'packs.$.inS3': true } }
_id: ObjectId(doc_id.toString()),
packs: { $elemMatch: { _id: pack_id, inS3: false } }
},
{ $set: { 'packs.$.inS3': true } },
{ projection: { 'packs.$': 1 } },
function (err, result) {
if (err != null) {
return callback(err)
}
if (result == null) {
if (!result.value) {
return callback(new Error('archive is not marked as progress'))
}
logger.log({ project_id, doc_id, pack_id }, 'marked as archived')
services/track-changes/app/js/PackWorker.js
+2 -2
View File
@@ -18,7 +18,7 @@ let project_id, doc_id
const Settings = require('settings-sharelatex')
const async = require('async')
const _ = require('underscore')
const { db, ObjectId } = require('./mongojs')
const { db, ObjectId } = require('./mongodb')
const fs = require('fs')
const Metrics = require('metrics-sharelatex')
Metrics.initialize('track-changes')
@@ -174,7 +174,7 @@ if (pending != null) {
_id: { $lt: ObjectIdFromDate(oneWeekAgo) },
last_checked: { $lt: oneWeekAgo }
},
{ _id: 1, doc_id: 1, project_id: 1 }
{ projection: { _id: 1, doc_id: 1, project_id: 1 } }
)
.sort({
last_checked: 1
services/track-changes/app/js/mongodb.js
+30
View File
@@ -0,0 +1,30 @@
const Settings = require('settings-sharelatex')
const { MongoClient, ObjectId } = require('mongodb')
const clientPromise = MongoClient.connect(
Settings.mongo.url,
Settings.mongo.options
)
let setupDbPromise
async function waitForDb() {
if (!setupDbPromise) {
setupDbPromise = setupDb()
}
await setupDbPromise
}
const db = {}
async function setupDb() {
const internalDb = (await clientPromise).db()
db.docHistory = internalDb.collection('docHistory')
db.docHistoryIndex = internalDb.collection('docHistoryIndex')
db.projectHistoryMetaData = internalDb.collection('projectHistoryMetaData')
}
module.exports = {
db,
ObjectId,
waitForDb
}
services/track-changes/config/settings.defaults.js
+4
View File
@@ -4,6 +4,10 @@ const TMP_DIR =
module.exports = {
mongo: {
options: {
useUnifiedTopology:
(process.env.MONGO_USE_UNIFIED_TOPOLOGY || 'true') === 'true'
},
url:
process.env.MONGO_CONNECTION_STRING ||
`mongodb://${process.env.MONGO_HOST || 'localhost'}/sharelatex`
services/track-changes/package-lock.json
Generated
+7 -7
View File
@@ -1496,9 +1496,9 @@
"integrity": "sha512-tbaUB1QpTIj4cKY8c1rvNAvEQXA+ekzHmbe4jzNfW3QWsF9GnnP/BRWyl6/qqS53heoYJ93naaFcm/jooONH8g=="
},
"bl": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/bl/-/bl-2.2.0.tgz",
"integrity": "sha512-wbgvOpqopSr7uq6fJrLH8EsvYMJf9gzfo2jCsL2eTy75qXPukA4pCgHamOQkZtY5vmfVtjB+P3LNlMHW5CEZXA==",
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/bl/-/bl-2.2.1.tgz",
"integrity": "sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g==",
"requires": {
"readable-stream": "^2.3.5",
"safe-buffer": "^5.1.1"
@@ -4717,12 +4717,12 @@
"integrity": "sha512-FehPVi2Dv7VPvAkLnN9haM1aarj1E9w08rkn2MAbbQJF5EbcOckdOHRAD9T35yUkfLVcs0YzYluNX4/+G8HaIw=="
},
"mongodb": {
"version": "3.5.5",
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.5.5.tgz",
"integrity": "sha512-GCjDxR3UOltDq00Zcpzql6dQo1sVry60OXJY3TDmFc2SWFY6c8Gn1Ardidc5jDirvJrx2GC3knGOImKphbSL3A==",
"version": "3.6.1",
"resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.1.tgz",
"integrity": "sha512-uH76Zzr5wPptnjEKJRQnwTsomtFOU/kQEU8a9hKHr2M7y9qVk7Q4Pkv0EQVp88742z9+RwvsdTw6dRjDZCNu1g==",
"requires": {
"bl": "^2.2.0",
"bson": "^1.1.1",
"bson": "^1.1.4",
"denque": "^1.4.1",
"require_optional": "^1.0.1",
"safe-buffer": "^5.1.2",
services/track-changes/package.json
+1
View File
@@ -30,6 +30,7 @@
"logger-sharelatex": "^2.2.0",
"metrics-sharelatex": "^2.6.2",
"mongo-uri": "^0.1.2",
"mongodb": "^3.6.0",
"mongojs": "3.1.0",
"redis": "~0.10.1",
"redis-sharelatex": "^1.0.13",
services/track-changes/test/acceptance/js/helpers/TrackChangesApp.js
+6 -4
View File
@@ -13,6 +13,7 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const app = require('../../../../app')
const { waitForDb } = require('../../../../app/js/mongodb')
require('logger-sharelatex')
const logger = require('logger-sharelatex')
const Settings = require('settings-sharelatex')
@@ -29,9 +30,10 @@ module.exports = {
return callback()
} else if (this.initing) {
return this.callbacks.push(callback)
} else {
this.initing = true
this.callbacks.push(callback)
}
this.initing = true
this.callbacks.push(callback)
waitForDb().then(() => {
return app.listen(
__guard__(
Settings.internal != null
@@ -56,7 +58,7 @@ module.exports = {
})()
}
)
}
})
}
}
function __guard__(value, transform) {
services/track-changes/test/unit/js/DocArchive/MongoAWS.js
+6 -4
View File
@@ -14,7 +14,7 @@ chai.should()
const sinon = require('sinon')
const modulePath = '../../../../app/js/MongoAWS.js'
const SandboxedModule = require('sandboxed-module')
const { ObjectId } = require('mongojs')
const { ObjectId } = require('mongodb')
const MemoryStream = require('memorystream')
const zlib = require('zlib')
@@ -44,7 +44,7 @@ describe('MongoAWS', function () {
'aws-sdk': (this.awssdk = {}),
fs: (this.fs = {}),
's3-streams': (this.S3S = {}),
'./mongojs': { db: (this.db = {}), ObjectId },
'./mongodb': { db: (this.db = {}), ObjectId },
JSONStream: (this.JSONStream = {}),
'readline-stream': (this.readline = sinon.stub()),
'metrics-sharelatex': { inc() {} }
@@ -92,7 +92,9 @@ describe('MongoAWS', function () {
this.S3S.ReadStream = () =>
MemoryStream.createReadStream(zbuf, { readable: true })
this.db.docHistory = {}
this.db.docHistory.insert = sinon.stub().callsArgWith(1, null, 'pack')
this.db.docHistory.insertOne = sinon
.stub()
.yields(null, { insertedId: ObjectId() })
return this.MongoAWS.unArchivePack(
this.project_id,
@@ -107,7 +109,7 @@ describe('MongoAWS', function () {
})
return it('should call db.docHistory.insert', function () {
return this.db.docHistory.insert.called.should.equal(true)
return this.db.docHistory.insertOne.called.should.equal(true)
})
})
})
services/track-changes/test/unit/js/MongoManager/MongoManagerTests.js
+6 -9
View File
@@ -16,7 +16,7 @@ const { expect } = chai
const modulePath = '../../../../app/js/MongoManager.js'
const packModulePath = '../../../../app/js/PackManager.js'
const SandboxedModule = require('sandboxed-module')
const { ObjectId } = require('mongojs')
const { ObjectId } = require('mongodb')
const tk = require('timekeeper')
describe('MongoManager', function () {
@@ -24,7 +24,7 @@ describe('MongoManager', function () {
tk.freeze(new Date())
this.MongoManager = SandboxedModule.require(modulePath, {
requires: {
'./mongojs': { db: (this.db = {}), ObjectId },
'./mongodb': { db: (this.db = {}), ObjectId },
'./PackManager': (this.PackManager = {}),
'metrics-sharelatex': { timeAsyncMethod() {} },
'logger-sharelatex': { log() {} }
@@ -156,7 +156,7 @@ describe('MongoManager', function () {
describe('backportProjectId', function () {
beforeEach(function () {
this.db.docHistory = { update: sinon.stub().callsArg(3) }
this.db.docHistory = { updateMany: sinon.stub().yields() }
return this.MongoManager.backportProjectId(
this.project_id,
this.doc_id,
@@ -165,7 +165,7 @@ describe('MongoManager', function () {
})
it("should insert the project_id into all entries for the doc_id which don't have it set", function () {
return this.db.docHistory.update
return this.db.docHistory.updateMany
.calledWith(
{
doc_id: ObjectId(this.doc_id),
@@ -173,9 +173,6 @@ describe('MongoManager', function () {
},
{
$set: { project_id: ObjectId(this.project_id) }
},
{
multi: true
}
)
.should.equal(true)
@@ -213,7 +210,7 @@ describe('MongoManager', function () {
beforeEach(function () {
this.metadata = { mock: 'metadata' }
this.db.projectHistoryMetaData = {
update: sinon.stub().callsArgWith(3, null, [this.metadata])
updateOne: sinon.stub().yields()
}
return this.MongoManager.setProjectMetaData(
this.project_id,
@@ -223,7 +220,7 @@ describe('MongoManager', function () {
})
it('should upsert the metadata into the DB', function () {
return this.db.projectHistoryMetaData.update
return this.db.projectHistoryMetaData.updateOne
.calledWith(
{
project_id: ObjectId(this.project_id)
services/track-changes/test/unit/js/PackManager/PackManagerTests.js
+9 -9
View File
@@ -17,7 +17,7 @@ const should = chai.should()
const { expect } = chai
const modulePath = '../../../../app/js/PackManager.js'
const SandboxedModule = require('sandboxed-module')
const { ObjectId } = require('mongojs')
const { ObjectId } = require('mongodb')
const _ = require('underscore')
const tk = require('timekeeper')
@@ -28,7 +28,7 @@ describe('PackManager', function () {
this.PackManager = SandboxedModule.require(modulePath, {
requires: {
bson: require('bson'),
'./mongojs': { db: (this.db = {}), ObjectId },
'./mongodb': { db: (this.db = {}), ObjectId },
'./LockManager': {},
'./MongoAWS': {},
'logger-sharelatex': { log: sinon.stub(), error: sinon.stub() },
@@ -65,7 +65,7 @@ describe('PackManager', function () {
{ op: 'op-4', meta: 'meta-4', v: 4 }
]
return (this.db.docHistory = {
save: sinon.stub().callsArg(1),
insertOne: sinon.stub().yields(),
insert: sinon.stub().callsArg(1),
updateOne: sinon.stub().yields(),
findAndModify: sinon.stub().callsArg(1)
@@ -390,7 +390,7 @@ describe('PackManager', function () {
return describe('for a small update that will expire', function () {
it('should insert the update into mongo', function () {
return this.db.docHistory.save
return this.db.docHistory.insertOne
.calledWithMatch({
pack: this.newUpdates,
project_id: ObjectId(this.project_id),
@@ -403,7 +403,7 @@ describe('PackManager', function () {
})
it('should set an expiry time in the future', function () {
return this.db.docHistory.save
return this.db.docHistory.insertOne
.calledWithMatch({
expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000)
})
@@ -496,7 +496,7 @@ describe('PackManager', function () {
return describe('for a small update that will not expire', function () {
it('should insert the update into mongo', function () {
return this.db.docHistory.save
return this.db.docHistory.insertOne
.calledWithMatch({
pack: this.newUpdates,
project_id: ObjectId(this.project_id),
@@ -509,7 +509,7 @@ describe('PackManager', function () {
})
it('should not set any expiry time', function () {
return this.db.docHistory.save
return this.db.docHistory.insertOne
.neverCalledWithMatch(sinon.match.has('expiresAt'))
.should.equal(true)
})
@@ -546,7 +546,7 @@ describe('PackManager', function () {
return describe('for a small update that will expire', function () {
it('should insert the update into mongo', function () {
return this.db.docHistory.save
return this.db.docHistory.insertOne
.calledWithMatch({
pack: this.newUpdates,
project_id: ObjectId(this.project_id),
@@ -559,7 +559,7 @@ describe('PackManager', function () {
})
it('should set an expiry time in the future', function () {
return this.db.docHistory.save
return this.db.docHistory.insertOne
.calledWithMatch({
expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000)
})
services/track-changes/test/unit/js/UpdatesManager/UpdatesManagerTests.js
+1 -1
View File
@@ -17,6 +17,7 @@ const sinon = require('sinon')
const chai = require('chai')
const should = chai.should()
const { expect } = chai
const { ObjectId } = require('mongodb')
const modulePath = '../../../../app/js/UpdatesManager.js'
const SandboxedModule = require('sandboxed-module')
@@ -864,7 +865,6 @@ describe('UpdatesManager', function () {
describe('fillUserInfo', function () {
describe('with valid users', function () {
beforeEach(function (done) {
const { ObjectId } = require('mongojs')
this.user_id_1 = ObjectId().toString()
this.user_id_2 = ObjectId().toString()
this.updates = [