From 4d3d43f44965f7e8a7f9ea0236fb9dcf110a0bae Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Mon, 24 Nov 2014 16:13:17 +0000 Subject: [PATCH 1/6] force an npm rebuild to work around issue #129, missing bcrypt bindings file see https://github.com/npm/npm/issues/5400 --- Gruntfile.coffee | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/Gruntfile.coffee b/Gruntfile.coffee index d9c5114557..cb75fed1c2 100644 --- a/Gruntfile.coffee +++ b/Gruntfile.coffee @@ -153,18 +153,22 @@ module.exports = (grunt) -> return callback(error) if error? Helpers.installNpmModules service, (error) -> return callback(error) if error? - Helpers.runGruntInstall service, (error) -> + Helpers.rebuildNpmModules service, (error) -> return callback(error) if error? - callback() + Helpers.runGruntInstall service, (error) -> + return callback(error) if error? + callback() updateService: (service, callback = (error) ->) -> Helpers.updateGitRepo service, (error) -> return callback(error) if error? Helpers.installNpmModules service, (error) -> return callback(error) if error? - Helpers.runGruntInstall service, (error) -> + Helpers.rebuildNpmModules service, (error) -> return callback(error) if error? - callback() + Helpers.runGruntInstall service, (error) -> + return callback(error) if error? + callback() cloneGitRepo: (service, callback = (error) ->) -> repo_src = service.repo @@ -207,13 +211,21 @@ module.exports = (grunt) -> proc = spawn "git", ["push", "--tags"], cwd: dir, stdio: "inherit" proc.on "close", () -> callback() - + installNpmModules: (service, callback = (error) ->) -> dir = service.name proc = spawn "npm", ["install"], stdio: "inherit", cwd: dir proc.on "close", () -> callback() - + + # work around for https://github.com/npm/npm/issues/5400 + # where binary modules are not built due to bug in npm + rebuildNpmModules: (service, callback = (error) ->) -> + dir = service.name + proc = spawn "npm", ["rebuild"], stdio: "inherit", cwd: dir + proc.on "close", () -> + callback() + createDataDirs: (callback = (error) ->) -> DIRS = [ "tmp/dumpFolder" From 77f1b582436192851a113b7ce25cf270c1130b7d Mon Sep 17 00:00:00 2001 From: James Allen Date: Thu, 4 Dec 2014 10:11:29 +0000 Subject: [PATCH 2/6] Update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 84dff1826b..7239983343 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,10 @@ The backend API for managing project tags (folders). An API for running server-side spelling checking on ShareLaTeX documents. +Dropbox +------- + +Please note that certain features like Dropbox integration are not functional in the open source code base yet, despite appearing in the user interface. We're working on this, sorry! Contributing ------------ From e8ebde8c40dd1d413b33556342c1b20d9cc1f86f Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Thu, 15 Jan 2015 12:30:42 +0000 Subject: [PATCH 3/6] added minimum memory requirements --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 7239983343..5592c47b73 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,8 @@ ShareLaTeX should run on OS X and Linux. You need: * A local instance of [Redis](http://redis.io/topics/quickstart) (version 2.6.12 or later) and [MongoDB](http://docs.mongodb.org/manual/installation/) running on their standard ports. * [TeXLive](https://www.tug.org/texlive/) 2013 or later with the `latexmk` program installed. +ShareLaTeX needs a minimum of 2gb of memory, it is likely to be more than that though depending on usage. + Other repositories ------------------ From 7e5c36021b87d7e2116b38ed0527f213d439a8b9 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 28 Jan 2015 17:50:39 +0000 Subject: [PATCH 4/6] mvp migration script --- Gruntfile.coffee | 18 ++++ migrations/1422460849371-doc-lines.coffee | 113 ++++++++++++++++++++++ package.json | 6 +- 3 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 migrations/1422460849371-doc-lines.coffee diff --git a/Gruntfile.coffee b/Gruntfile.coffee index d9c5114557..61be652c2d 100644 --- a/Gruntfile.coffee +++ b/Gruntfile.coffee @@ -55,6 +55,8 @@ module.exports = (grunt) -> grunt.loadNpmTasks 'grunt-execute' grunt.loadNpmTasks 'grunt-available-tasks' grunt.loadNpmTasks 'grunt-concurrent' + grunt.loadNpmTasks "grunt-contrib-coffee" + execute = {} for service in SERVICES @@ -70,6 +72,18 @@ module.exports = (grunt) -> options: limit: SERVICES.length logConcurrentOutput: true + coffee: + migrate: + expand: true, + flatten: false, + cwd: './', + src: ['./migrations/*.coffee'], + dest: './', + ext: '.js' + options: + bare:true + + availabletasks: tasks: @@ -147,6 +161,10 @@ module.exports = (grunt) -> grunt.registerTask "build:upstart_scripts", "Create upstart scripts for each service", () -> Helpers.buildUpstartScripts() + + grunt.registerTask 'migrate', 'run migrations', ['coffee:migrate'] + + Helpers = installService: (service, callback = (error) ->) -> Helpers.cloneGitRepo service, (error) -> diff --git a/migrations/1422460849371-doc-lines.coffee b/migrations/1422460849371-doc-lines.coffee new file mode 100644 index 0000000000..1d370e986f --- /dev/null +++ b/migrations/1422460849371-doc-lines.coffee @@ -0,0 +1,113 @@ + +fs = require("fs") +mongojs = require("mongojs") +ObjectId = mongojs.ObjectId +db = mongojs('sharelatex', ['projects', 'docs']) +_ = require("underscore") +async = require("async") + +local_db_path = "/tmp/process-db" + +Datastore = require('nedb') +processDb = new Datastore({ filename: local_db_path}) + + +printProgress = -> + processDb.count {processed:false}, (err, todo)-> + processDb.count {}, (err, total)-> + console.log "#{todo}/#{total} processed" + setTimeout printProgress, 1000 + + +writeProjectIdsToDisk = (callback)-> + db.projects.find {}, {_id:1}, (err,ids)-> + console.log "total found projects in mongo #{ids.length}" + ids = _.map ids, (id)-> return id._id.toString() + jobs = _.map ids, (id)-> + return (cb)-> + processDb.findOne {project_id:id}, (err, doc)-> + if doc? + return cb() + processDb.insert {project_id:id, processed:false}, cb + async.series jobs, (err)-> + processDb.count {processed:false}, (err, count)-> + console.log "projects to process: #{count}" + callback() + +getNextProjectToProccess = (callback)-> + processDb.findOne {processed:false}, (err, doc)-> + callback err, doc.project_id + +markProjectAsProcessed = (project_id, callback)-> + processDb.update project_id:project_id, {$set:{processed:true}}, {}, callback + + +getAllDocs = (project_id, callback = (error, docs) ->) -> + db.projects.findOne _id:ObjectId(project_id), (error, project) -> + return callback(error) if error? + return callback new Errors.NotFoundError("No such project: #{project_id}") if !project? + findAllDocsInProject project, (error, docs) -> + return callback(error) if error? + return callback null, docs + +findAllDocsInProject = (project, callback = (error, docs) ->) -> + callback null, _findAllDocsInFolder project.rootFolder[0] + +_findDocInFolder = (folder = {}, doc_id, currentPath) -> + for doc, i in folder.docs or [] + if doc?._id? and doc._id.toString() == doc_id.toString() + return { + doc: doc + mongoPath: "#{currentPath}.docs.#{i}" + } + + for childFolder, i in folder.folders or [] + result = _findDocInFolder childFolder, doc_id, "#{currentPath}.folders.#{i}" + return result if result? + + return null + +_findAllDocsInFolder = (folder = {}) -> + docs = folder.docs or [] + for childFolder in folder.folders or [] + docs = docs.concat _findAllDocsInFolder childFolder + return docs + +insertDocIntoDocCollection = (project_id, doc_id, lines, oldRev, callback)-> + update = {} + update["_id"] = ObjectId(doc_id) + update["lines"] = lines + update["project_id"] = ObjectId(project_id) + update["rev"] = oldRev + db.docs.insert _id: ObjectId(doc_id), callback + +saveDocsIntoMongo = (project_id, docs, callback)-> + jobs = _.map docs, (doc)-> + (cb)-> + insertDocIntoDocCollection project_id, doc._id, project_id.lines, doc.rev, cb + async.series jobs, callback + + +processNext = -> + processDb.count {processed:false}, (err, total)-> + if total == 0 + console.log "DONE" + process.exit() + else + getNextProjectToProccess (err, project_id)-> + getAllDocs project_id, (err, docs)-> + saveDocsIntoMongo project_id, docs, -> + markProjectAsProcessed project_id, -> + processNext() + +processDb.loadDatabase -> + writeProjectIdsToDisk -> + processNext() + +exports.up = (next)-> + + next() + + +exports.down = (next)-> + next() diff --git a/package.json b/package.json index 2e922d5b14..fdb5894911 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,11 @@ "description": "An online collaborative LaTeX editor", "dependencies": { "async": "^0.9.0", + "mongojs": "^0.18.1", + "nedb": "^1.1.1", "rimraf": "~2.2.6", - "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git" + "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git", + "underscore": "^1.7.0" }, "devDependencies": { "grunt": "~0.4.2", @@ -14,6 +17,7 @@ "grunt-execute": "~0.1.5", "grunt-available-tasks": "~0.4.1", "grunt-concurrent": "~0.4.3", + "grunt-contrib-coffee": "~0.10.1", "semver": "~2.2.1", "knox": "~0.8.9" } From 59307aa94945a4851b68bab55f1777b5bc33a21c Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Thu, 29 Jan 2015 11:37:52 +0000 Subject: [PATCH 5/6] moved to a file appending with grep data store --- migrations/1422460849371-doc-lines.coffee | 83 ++++++++++++----------- package.json | 2 +- 2 files changed, 43 insertions(+), 42 deletions(-) diff --git a/migrations/1422460849371-doc-lines.coffee b/migrations/1422460849371-doc-lines.coffee index 1d370e986f..07ed0f8d6b 100644 --- a/migrations/1422460849371-doc-lines.coffee +++ b/migrations/1422460849371-doc-lines.coffee @@ -3,44 +3,33 @@ fs = require("fs") mongojs = require("mongojs") ObjectId = mongojs.ObjectId db = mongojs('sharelatex', ['projects', 'docs']) -_ = require("underscore") +_ = require("lodash") async = require("async") +exec = require("child_process").exec -local_db_path = "/tmp/process-db" - -Datastore = require('nedb') -processDb = new Datastore({ filename: local_db_path}) +finished_projects_path = "/tmp/finished-projects" printProgress = -> - processDb.count {processed:false}, (err, todo)-> - processDb.count {}, (err, total)-> - console.log "#{todo}/#{total} processed" - setTimeout printProgress, 1000 + exec "wc #{finished_projects_path}", (error, results) -> + #console.log results + setTimeout printProgress, 1000 * 30 +checkIfFileHasBeenProccessed = (project_id, callback)-> + exec "grep #{project_id} #{finished_projects_path}", (error, results) -> + hasBeenProcessed = _.include(results, project_id) + console.log hasBeenProcessed, project_id + callback(null, hasBeenProcessed) -writeProjectIdsToDisk = (callback)-> +getProjectIds = (callback)-> + console.log "finding all project id's - #{new Date().toString()}" db.projects.find {}, {_id:1}, (err,ids)-> - console.log "total found projects in mongo #{ids.length}" + console.log "total found projects in mongo #{ids.length} - #{new Date().toString()}" ids = _.map ids, (id)-> return id._id.toString() - jobs = _.map ids, (id)-> - return (cb)-> - processDb.findOne {project_id:id}, (err, doc)-> - if doc? - return cb() - processDb.insert {project_id:id, processed:false}, cb - async.series jobs, (err)-> - processDb.count {processed:false}, (err, count)-> - console.log "projects to process: #{count}" - callback() - -getNextProjectToProccess = (callback)-> - processDb.findOne {processed:false}, (err, doc)-> - callback err, doc.project_id + callback(err, ids) markProjectAsProcessed = (project_id, callback)-> - processDb.update project_id:project_id, {$set:{processed:true}}, {}, callback - + fs.appendFile finished_projects_path, "#{project_id}\n", callback getAllDocs = (project_id, callback = (error, docs) ->) -> db.projects.findOne _id:ObjectId(project_id), (error, project) -> @@ -88,21 +77,33 @@ saveDocsIntoMongo = (project_id, docs, callback)-> async.series jobs, callback -processNext = -> - processDb.count {processed:false}, (err, total)-> - if total == 0 - console.log "DONE" - process.exit() - else - getNextProjectToProccess (err, project_id)-> - getAllDocs project_id, (err, docs)-> - saveDocsIntoMongo project_id, docs, -> - markProjectAsProcessed project_id, -> - processNext() +processNext = (project_id, callback)-> + #console.log("starting to process #{project_id} - #{new Date().toString()}") + checkIfFileHasBeenProccessed project_id, (err, hasBeenProcessed)-> + if hasBeenProcessed + return callback() + getAllDocs project_id, (err, docs)-> + if err? + console.error err, project_id, "could not get all docs" + return callback() + saveDocsIntoMongo project_id, docs, -> + if err? + console.error err, project_id, "could not save docs into mongo" + return callback() + markProjectAsProcessed project_id, -> + callback() -processDb.loadDatabase -> - writeProjectIdsToDisk -> - processNext() +getProjectIds (err, ids)-> + printProgress() + jobs = _.map ids, (id)-> + return (cb)-> + processNext(id, cb) + async.series jobs, (err)-> + if err? + console.error err, "at end of jobs" + else + console.log "finished" + process.exit() exports.up = (next)-> diff --git a/package.json b/package.json index fdb5894911..a50146f3ef 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,8 @@ "description": "An online collaborative LaTeX editor", "dependencies": { "async": "^0.9.0", + "lodash": "^3.0.0", "mongojs": "^0.18.1", - "nedb": "^1.1.1", "rimraf": "~2.2.6", "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git", "underscore": "^1.7.0" From 2bd88e5319571058df58649f7e3defbb261a672f Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Thu, 29 Jan 2015 12:20:49 +0000 Subject: [PATCH 6/6] added some null checks and more persistance --- migrations/1422460849371-doc-lines.coffee | 35 +++++++++++++++++------ 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/migrations/1422460849371-doc-lines.coffee b/migrations/1422460849371-doc-lines.coffee index 07ed0f8d6b..a97d2907ac 100644 --- a/migrations/1422460849371-doc-lines.coffee +++ b/migrations/1422460849371-doc-lines.coffee @@ -8,7 +8,7 @@ async = require("async") exec = require("child_process").exec finished_projects_path = "/tmp/finished-projects" - +all_projects_path = "/tmp/all-projects" printProgress = -> exec "wc #{finished_projects_path}", (error, results) -> @@ -18,15 +18,32 @@ printProgress = -> checkIfFileHasBeenProccessed = (project_id, callback)-> exec "grep #{project_id} #{finished_projects_path}", (error, results) -> hasBeenProcessed = _.include(results, project_id) - console.log hasBeenProcessed, project_id + #console.log hasBeenProcessed, project_id callback(null, hasBeenProcessed) -getProjectIds = (callback)-> +loadProjectIds = (callback)-> + fs.readFile all_projects_path, "utf-8", (err, data)-> + console.log data.length + ids = data.split("\n") + console.log ids.length + callback err, ids + +getAndWriteProjectids = (callback)-> console.log "finding all project id's - #{new Date().toString()}" - db.projects.find {}, {_id:1}, (err,ids)-> + db.projects.find {}, {_id:1}, (err, ids)-> console.log "total found projects in mongo #{ids.length} - #{new Date().toString()}" - ids = _.map ids, (id)-> return id._id.toString() - callback(err, ids) + ids = _.pluck ids, '_id' + ids = _.filter ids, (id)-> id? + fileData = ids.join("\n") + fs.writeFile all_projects_path, fileData, -> + callback(err, ids) + +getProjectIds = (callback)-> + exists = fs.existsSync all_projects_path + if exists + loadProjectIds callback + else + getAndWriteProjectids callback markProjectAsProcessed = (project_id, callback)-> fs.appendFile finished_projects_path, "#{project_id}\n", callback @@ -34,7 +51,9 @@ markProjectAsProcessed = (project_id, callback)-> getAllDocs = (project_id, callback = (error, docs) ->) -> db.projects.findOne _id:ObjectId(project_id), (error, project) -> return callback(error) if error? - return callback new Errors.NotFoundError("No such project: #{project_id}") if !project? + if !project? + console.error("No such project: #{project_id}") + return callback("no such project #{project_id}") findAllDocsInProject project, (error, docs) -> return callback(error) if error? return callback null, docs @@ -78,9 +97,9 @@ saveDocsIntoMongo = (project_id, docs, callback)-> processNext = (project_id, callback)-> - #console.log("starting to process #{project_id} - #{new Date().toString()}") checkIfFileHasBeenProccessed project_id, (err, hasBeenProcessed)-> if hasBeenProcessed + console.log "#{project_id} already procssed, skipping" return callback() getAllDocs project_id, (err, docs)-> if err?