diff --git a/Gruntfile.coffee b/Gruntfile.coffee index cb05597a24..a1633cd000 100644 --- a/Gruntfile.coffee +++ b/Gruntfile.coffee @@ -55,6 +55,8 @@ module.exports = (grunt) -> grunt.loadNpmTasks 'grunt-execute' grunt.loadNpmTasks 'grunt-available-tasks' grunt.loadNpmTasks 'grunt-concurrent' + grunt.loadNpmTasks "grunt-contrib-coffee" + execute = {} for service in SERVICES @@ -70,6 +72,18 @@ module.exports = (grunt) -> options: limit: SERVICES.length logConcurrentOutput: true + coffee: + migrate: + expand: true, + flatten: false, + cwd: './', + src: ['./migrations/*.coffee'], + dest: './', + ext: '.js' + options: + bare:true + + availabletasks: tasks: @@ -147,24 +161,32 @@ module.exports = (grunt) -> grunt.registerTask "build:upstart_scripts", "Create upstart scripts for each service", () -> Helpers.buildUpstartScripts() + + grunt.registerTask 'migrate', 'run migrations', ['coffee:migrate'] + + Helpers = installService: (service, callback = (error) ->) -> Helpers.cloneGitRepo service, (error) -> return callback(error) if error? Helpers.installNpmModules service, (error) -> return callback(error) if error? - Helpers.runGruntInstall service, (error) -> + Helpers.rebuildNpmModules service, (error) -> return callback(error) if error? - callback() + Helpers.runGruntInstall service, (error) -> + return callback(error) if error? + callback() updateService: (service, callback = (error) ->) -> Helpers.updateGitRepo service, (error) -> return callback(error) if error? Helpers.installNpmModules service, (error) -> return callback(error) if error? - Helpers.runGruntInstall service, (error) -> + Helpers.rebuildNpmModules service, (error) -> return callback(error) if error? - callback() + Helpers.runGruntInstall service, (error) -> + return callback(error) if error? + callback() cloneGitRepo: (service, callback = (error) ->) -> repo_src = service.repo @@ -207,13 +229,21 @@ module.exports = (grunt) -> proc = spawn "git", ["push", "--tags"], cwd: dir, stdio: "inherit" proc.on "close", () -> callback() - + installNpmModules: (service, callback = (error) ->) -> dir = service.name proc = spawn "npm", ["install"], stdio: "inherit", cwd: dir proc.on "close", () -> callback() - + + # work around for https://github.com/npm/npm/issues/5400 + # where binary modules are not built due to bug in npm + rebuildNpmModules: (service, callback = (error) ->) -> + dir = service.name + proc = spawn "npm", ["rebuild"], stdio: "inherit", cwd: dir + proc.on "close", () -> + callback() + createDataDirs: (callback = (error) ->) -> DIRS = [ "tmp/dumpFolder" diff --git a/README.md b/README.md index 84dff1826b..5592c47b73 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,8 @@ ShareLaTeX should run on OS X and Linux. You need: * A local instance of [Redis](http://redis.io/topics/quickstart) (version 2.6.12 or later) and [MongoDB](http://docs.mongodb.org/manual/installation/) running on their standard ports. * [TeXLive](https://www.tug.org/texlive/) 2013 or later with the `latexmk` program installed. +ShareLaTeX needs a minimum of 2gb of memory, it is likely to be more than that though depending on usage. + Other repositories ------------------ @@ -78,6 +80,10 @@ The backend API for managing project tags (folders). An API for running server-side spelling checking on ShareLaTeX documents. +Dropbox +------- + +Please note that certain features like Dropbox integration are not functional in the open source code base yet, despite appearing in the user interface. We're working on this, sorry! Contributing ------------ diff --git a/migrations/1422460849371-doc-lines.coffee b/migrations/1422460849371-doc-lines.coffee new file mode 100644 index 0000000000..a97d2907ac --- /dev/null +++ b/migrations/1422460849371-doc-lines.coffee @@ -0,0 +1,133 @@ + +fs = require("fs") +mongojs = require("mongojs") +ObjectId = mongojs.ObjectId +db = mongojs('sharelatex', ['projects', 'docs']) +_ = require("lodash") +async = require("async") +exec = require("child_process").exec + +finished_projects_path = "/tmp/finished-projects" +all_projects_path = "/tmp/all-projects" + +printProgress = -> + exec "wc #{finished_projects_path}", (error, results) -> + #console.log results + setTimeout printProgress, 1000 * 30 + +checkIfFileHasBeenProccessed = (project_id, callback)-> + exec "grep #{project_id} #{finished_projects_path}", (error, results) -> + hasBeenProcessed = _.include(results, project_id) + #console.log hasBeenProcessed, project_id + callback(null, hasBeenProcessed) + +loadProjectIds = (callback)-> + fs.readFile all_projects_path, "utf-8", (err, data)-> + console.log data.length + ids = data.split("\n") + console.log ids.length + callback err, ids + +getAndWriteProjectids = (callback)-> + console.log "finding all project id's - #{new Date().toString()}" + db.projects.find {}, {_id:1}, (err, ids)-> + console.log "total found projects in mongo #{ids.length} - #{new Date().toString()}" + ids = _.pluck ids, '_id' + ids = _.filter ids, (id)-> id? + fileData = ids.join("\n") + fs.writeFile all_projects_path, fileData, -> + callback(err, ids) + +getProjectIds = (callback)-> + exists = fs.existsSync all_projects_path + if exists + loadProjectIds callback + else + getAndWriteProjectids callback + +markProjectAsProcessed = (project_id, callback)-> + fs.appendFile finished_projects_path, "#{project_id}\n", callback + +getAllDocs = (project_id, callback = (error, docs) ->) -> + db.projects.findOne _id:ObjectId(project_id), (error, project) -> + return callback(error) if error? + if !project? + console.error("No such project: #{project_id}") + return callback("no such project #{project_id}") + findAllDocsInProject project, (error, docs) -> + return callback(error) if error? + return callback null, docs + +findAllDocsInProject = (project, callback = (error, docs) ->) -> + callback null, _findAllDocsInFolder project.rootFolder[0] + +_findDocInFolder = (folder = {}, doc_id, currentPath) -> + for doc, i in folder.docs or [] + if doc?._id? and doc._id.toString() == doc_id.toString() + return { + doc: doc + mongoPath: "#{currentPath}.docs.#{i}" + } + + for childFolder, i in folder.folders or [] + result = _findDocInFolder childFolder, doc_id, "#{currentPath}.folders.#{i}" + return result if result? + + return null + +_findAllDocsInFolder = (folder = {}) -> + docs = folder.docs or [] + for childFolder in folder.folders or [] + docs = docs.concat _findAllDocsInFolder childFolder + return docs + +insertDocIntoDocCollection = (project_id, doc_id, lines, oldRev, callback)-> + update = {} + update["_id"] = ObjectId(doc_id) + update["lines"] = lines + update["project_id"] = ObjectId(project_id) + update["rev"] = oldRev + db.docs.insert _id: ObjectId(doc_id), callback + +saveDocsIntoMongo = (project_id, docs, callback)-> + jobs = _.map docs, (doc)-> + (cb)-> + insertDocIntoDocCollection project_id, doc._id, project_id.lines, doc.rev, cb + async.series jobs, callback + + +processNext = (project_id, callback)-> + checkIfFileHasBeenProccessed project_id, (err, hasBeenProcessed)-> + if hasBeenProcessed + console.log "#{project_id} already procssed, skipping" + return callback() + getAllDocs project_id, (err, docs)-> + if err? + console.error err, project_id, "could not get all docs" + return callback() + saveDocsIntoMongo project_id, docs, -> + if err? + console.error err, project_id, "could not save docs into mongo" + return callback() + markProjectAsProcessed project_id, -> + callback() + +getProjectIds (err, ids)-> + printProgress() + jobs = _.map ids, (id)-> + return (cb)-> + processNext(id, cb) + async.series jobs, (err)-> + if err? + console.error err, "at end of jobs" + else + console.log "finished" + process.exit() + +exports.up = (next)-> + + next() + + +exports.down = (next)-> + next() diff --git a/package.json b/package.json index 7165d80bf6..ffbfa4a2e8 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,11 @@ "description": "An online collaborative LaTeX editor", "dependencies": { "async": "^0.9.0", + "lodash": "^3.0.0", + "mongojs": "^0.18.1", "rimraf": "~2.2.6", - "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git" + "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git", + "underscore": "^1.7.0" }, "devDependencies": { "grunt": "~0.4.2", @@ -14,6 +17,7 @@ "grunt-execute": "~0.1.5", "grunt-available-tasks": "~0.4.1", "grunt-concurrent": "~0.4.3", + "grunt-contrib-coffee": "~0.10.1", "semver": "~2.2.1", "knox": "~0.8.9" }