From 5ea12c3de8e2ff5eeaf9812cbcd1633334d9e25c Mon Sep 17 00:00:00 2001 From: James Allen Date: Fri, 20 Mar 2015 20:32:07 +0000 Subject: [PATCH 1/9] Update CHANGELOG.md --- CHANGELOG.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc8167022b..baac4bcc25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,23 @@ +v0.1.4 +------ + +* Move to a private registration scheme where users must be added by an admin. +* Proxy websockets connection through web to real-time service so no websocketsUrl parameter is needed. +* Use worker aspell processes in spelling to prevent excessing forking. +* Properly clean up after long running ImageMagick conversions in the filestore. +* Allow a configurable app name and email contact address. +* Switch to new PDF viewer with partial page loading for immediate preview of visible page. + +v0.1.3 +------ + +* Fix bug with large files being corrupted when downloaded. +* Update Ace editor to lastest release. +* Lots of added null checks in the front-end javascript. +* Don't crash if 'unzip' program isn't present. +* Allow track-changes history to be packed into compressed 'packs'. This must be done manually for now. +* Escape any shell special characters in the CLSI root path. + v0.1.2 ------ From 5dd40dc7cb1559aa847f74f4fef8855e5673e9f5 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 14 Apr 2015 15:21:07 +0100 Subject: [PATCH 2/9] added migration 2 to clean up projects, removing the doc lines --- .../2_doc_lines_delete_from_project.coffee | 194 ++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 migrations/2_doc_lines_delete_from_project.coffee diff --git a/migrations/2_doc_lines_delete_from_project.coffee b/migrations/2_doc_lines_delete_from_project.coffee new file mode 100644 index 0000000000..49e7eb0918 --- /dev/null +++ b/migrations/2_doc_lines_delete_from_project.coffee @@ -0,0 +1,194 @@ +Settings = require "settings-sharelatex" +fs = require("fs") +mongojs = require("mongojs") +ObjectId = mongojs.ObjectId +console.log Settings.mongo.url +db = mongojs(Settings.mongo.url, ['projects', 'docs']) +_ = require("lodash") +async = require("async") +exec = require("child_process").exec + +finished_projects_path = "/home/sharelatex/finished-projects" +all_projects_path = "/home/sharelatex/all-projects" +unmigrated_docs_path = "/home/sharelatex/unmigrated" + +processedFiles = fs.readFileSync finished_projects_path + +printProgress = -> + exec "wc #{finished_projects_path}", (error, results) -> + setTimeout printProgress, 1000 * 30 + +checkIfFileHasBeenProccessed = (project_id, callback)-> + hasBeenProcessed = _.include processedFiles, project + callback null, hasBeenProcessed + # exec "grep #{project_id} #{finished_projects_path}", (error, results) -> + # hasBeenProcessed = _.include(results, project_id) + # callback(error, hasBeenProcessed) + +loadProjectIds = (callback)-> + console.log "loading project ids from #{all_projects_path}" + fs.readFile all_projects_path, "utf-8", (err, data)-> + ids = data.split("\n") + console.log "loaded #{ids.length} project ids from #{all_projects_path}" + callback err, ids + +getAndWriteProjectids = (callback)-> + console.log "finding all project id's - #{new Date().toString()}" + db.projects.find {}, {_id:1}, (err, ids)-> + console.log "total found projects in mongo #{ids.length} - #{new Date().toString()}" + ids = _.pluck ids, '_id' + ids = _.filter ids, (id)-> id? + fileData = ids.join("\n") + fs.writeFile all_projects_path, fileData, -> + callback(err, ids) + +markDocAsUnmigrated = (project_id, doc_id, callback)-> + console.log "#{project_id} #{doc_id} unmigrated" + markProjectAsProcessed project_id, (err)-> + fs.appendFile unmigrated_docs_path, "#{project_id} #{doc_id}\n", callback + +markUnmigratedDocs = (project_id, docs, callback)-> + console.log docs.length, project_id, "unmigrated" + jobs = _.map docs, (doc)-> + (cb)-> + markDocAsUnmigrated project_id, doc._id, cb + async.series jobs, callback + +getProjectIds = (callback)-> + exists = fs.existsSync all_projects_path + if exists + loadProjectIds callback + else + getAndWriteProjectids callback + +markProjectAsProcessed = (project_id, callback)-> + fs.appendFile finished_projects_path, "#{project_id}\n", callback + +getAllDocs = (project_id, callback = (error, docs) ->) -> + excludes = {} + for i in [0..12] + excludes["rootFolder#{Array(i).join(".folders")}.docs.lines"] = 0 + db.projects.findOne _id: ObjectId(project_id.toString()), excludes, (error, project) -> + return callback(error) if error? + if !project? + console.log "no such project #{project_id}" + return callback() + findAllDocsInProject project, (error, docs) -> + return callback(error) if error? + return callback null, docs, project + +findAllDocsInProject = (project, callback = (error, docs) ->) -> + callback null, _findAllDocsInFolder project.rootFolder[0] + +findDocInProject = (project, doc_id, callback = (error, doc, mongoPath) ->) -> + result = _findDocInFolder project.rootFolder[0], doc_id, "rootFolder.0" + if result? + callback null, result.doc, result.mongoPath + else + callback null, null, null + +_findDocInFolder = (folder = {}, doc_id, currentPath) -> + for doc, i in folder.docs or [] + if doc?._id? and doc._id.toString() == doc_id.toString() + return { + doc: doc + mongoPath: "#{currentPath}.docs.#{i}" + } + for childFolder, i in folder.folders or [] + result = _findDocInFolder childFolder, doc_id, "#{currentPath}.folders.#{i}" + return result if result? + + return null + +_findAllDocsInFolder = (folder = {}) -> + docs = folder.docs or [] + for childFolder in folder.folders or [] + docs = docs.concat _findAllDocsInFolder childFolder + return docs + +isDocInDocCollection = (doc, callback)-> + if !doc?._id? or doc._id.length == 0 + return callback(null, true) + db.docs.find({_id: ObjectId(doc._id+"")}, {_id: 1}).limit 1, (err, foundDocs)-> + exists = foundDocs.length > 0 + console.log doc._id, "Exits = "+exists + callback err, exists + +getWhichDocsCanBeDeleted = (docs, callback = (err, docsToBeDeleted, unmigratedDocs)->)-> + docsToBeDeleted = [] + unmigratedDocs = [] + + jobs = _.map docs, (doc)-> + return (cb)-> + isDocInDocCollection doc, (err, exists)-> + if exists + docsToBeDeleted.push doc + else + unmigratedDocs.push doc + cb(err) + async.series jobs, (err)-> + callback err, docsToBeDeleted, unmigratedDocs + +whipeDocLines = (project_id, mongoPath, callback)-> + update = + $unset: {} + update.$unset["#{mongoPath}.lines"] = "" + update.$unset["#{mongoPath}.rev"] = "" + console.log "project id = #{ObjectId(project_id+'')}", update + #callback() + db.projects.update _id: ObjectId(project_id+''), update, callback + + +removeDocLinesFromProject = (docs, project, callback)-> + jobs = _.map docs, (doc)-> + (cb)-> + findDocInProject project, doc._id, (err, doc, mongoPath)-> + whipeDocLines project._id, mongoPath, cb + async.parallelLimit jobs, 5, callback + +processNext = (project_id, callback)-> + if !project_id? or project_id.length == 0 + return callback() + checkIfFileHasBeenProccessed project_id, (err, hasBeenProcessed)-> + if hasBeenProcessed + console.log "#{project_id} already procssed, skipping" + return callback() + console.log "#{project_id} processing" + getAllDocs project_id, (err, docs, project)-> + if err? + console.error err, project_id, "could not get all docs" + return callback(err) + else + getWhichDocsCanBeDeleted docs, (err, docsToBeDeleted, unmigratedDocs)-> + if err? + console.error err, project_id, "could not save docs into mongo" + return callback(err) + markUnmigratedDocs project_id, unmigratedDocs, (err)-> + removeDocLinesFromProject docsToBeDeleted, project, (err)-> + if err? + return callback(err) + markProjectAsProcessed project_id, (err)-> + setTimeout( + -> callback(err) + ,5000) + +exports.migrate = (client, done = ->)-> + getProjectIds (err, ids)-> + printProgress() + jobs = _.map ids, (id)-> + return (cb)-> + processNext(id, cb) + async.series jobs, (err)-> + if err? + console.error err, "at end of jobs" + process.exit() + else + console.log "finished" + process.exit() + done(err) + + +exports.rollback = (next)-> + next() + +exports.migrate() From ce10ceb2d94173817500939648f3a9135eca6841 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Thu, 16 Apr 2015 13:45:16 +0100 Subject: [PATCH 3/9] improved migration script to delete docs --- .../2_doc_lines_delete_from_project.coffee | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/migrations/2_doc_lines_delete_from_project.coffee b/migrations/2_doc_lines_delete_from_project.coffee index 49e7eb0918..bc413d8134 100644 --- a/migrations/2_doc_lines_delete_from_project.coffee +++ b/migrations/2_doc_lines_delete_from_project.coffee @@ -8,22 +8,19 @@ _ = require("lodash") async = require("async") exec = require("child_process").exec -finished_projects_path = "/home/sharelatex/finished-projects" -all_projects_path = "/home/sharelatex/all-projects" -unmigrated_docs_path = "/home/sharelatex/unmigrated" +finished_projects_path = "./finished-projects" +all_projects_path = "./all-projects" +unmigrated_docs_path = "./unmigrated" -processedFiles = fs.readFileSync finished_projects_path printProgress = -> exec "wc #{finished_projects_path}", (error, results) -> setTimeout printProgress, 1000 * 30 checkIfFileHasBeenProccessed = (project_id, callback)-> - hasBeenProcessed = _.include processedFiles, project - callback null, hasBeenProcessed - # exec "grep #{project_id} #{finished_projects_path}", (error, results) -> - # hasBeenProcessed = _.include(results, project_id) - # callback(error, hasBeenProcessed) + exec "grep #{project_id} #{finished_projects_path}", (error, results) -> + hasBeenProcessed = _.include(results, project_id) + callback(error, hasBeenProcessed) loadProjectIds = (callback)-> console.log "loading project ids from #{all_projects_path}" @@ -191,4 +188,3 @@ exports.migrate = (client, done = ->)-> exports.rollback = (next)-> next() -exports.migrate() From e3b3b8572697c9742e456c29d09e1e6154297ad6 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 16 Sep 2015 13:41:09 +0100 Subject: [PATCH 4/9] add the migrate task into grunt --- Gruntfile.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gruntfile.coffee b/Gruntfile.coffee index d82427221c..9ed6ff9275 100644 --- a/Gruntfile.coffee +++ b/Gruntfile.coffee @@ -165,7 +165,7 @@ module.exports = (grunt) -> Helpers.buildUpstartScripts() - #grunt.registerTask 'migrate', "compile migrations and run them", ['coffee:migrate', 'shell:migrate'] + grunt.registerTask 'migrate', "compile migrations and run them", ['coffee:migrate', 'shell:migrate'] Helpers = From feafbb54162f37a5ccdf1def799814ff6b1c6cf4 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 16 Sep 2015 14:22:54 +0100 Subject: [PATCH 5/9] renamed first migration to move_doc_lines_to_doc_collection --- ...doc_lines.coffee => 1_move_doc_lines_to_doc_collection.coffee} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename migrations/{1_doc_lines.coffee => 1_move_doc_lines_to_doc_collection.coffee} (100%) diff --git a/migrations/1_doc_lines.coffee b/migrations/1_move_doc_lines_to_doc_collection.coffee similarity index 100% rename from migrations/1_doc_lines.coffee rename to migrations/1_move_doc_lines_to_doc_collection.coffee From 1e073da1fdb1c6fca4364d8d68e0df31285eb4ad Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 16 Sep 2015 14:44:53 +0100 Subject: [PATCH 6/9] bring down migration timeout in delete doc lines to 0ms --- migrations/2_doc_lines_delete_from_project.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/2_doc_lines_delete_from_project.coffee b/migrations/2_doc_lines_delete_from_project.coffee index bc413d8134..ee0b74b914 100644 --- a/migrations/2_doc_lines_delete_from_project.coffee +++ b/migrations/2_doc_lines_delete_from_project.coffee @@ -167,7 +167,7 @@ processNext = (project_id, callback)-> markProjectAsProcessed project_id, (err)-> setTimeout( -> callback(err) - ,5000) + ,0) exports.migrate = (client, done = ->)-> getProjectIds (err, ids)-> From 0980b2e6dc83ea4b9d80df7c2794457bf2b0c63d Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 16 Sep 2015 15:58:38 +0100 Subject: [PATCH 7/9] change timeout in migrations-1 to 0ms --- migrations/1_move_doc_lines_to_doc_collection.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/1_move_doc_lines_to_doc_collection.coffee b/migrations/1_move_doc_lines_to_doc_collection.coffee index 9589b67f4c..d1f94638b3 100644 --- a/migrations/1_move_doc_lines_to_doc_collection.coffee +++ b/migrations/1_move_doc_lines_to_doc_collection.coffee @@ -139,7 +139,7 @@ processNext = (project_id, callback)-> markProjectAsProcessed project_id, (err)-> setTimeout( -> callback(err) - ,500) + ,0) From 6aa2446eea375b4f0aaf768e014ddd086a26166c Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 16 Sep 2015 16:26:51 +0100 Subject: [PATCH 8/9] remove process.exit from migrations --- migrations/2_doc_lines_delete_from_project.coffee | 2 -- 1 file changed, 2 deletions(-) diff --git a/migrations/2_doc_lines_delete_from_project.coffee b/migrations/2_doc_lines_delete_from_project.coffee index ee0b74b914..96c8d5b334 100644 --- a/migrations/2_doc_lines_delete_from_project.coffee +++ b/migrations/2_doc_lines_delete_from_project.coffee @@ -178,10 +178,8 @@ exports.migrate = (client, done = ->)-> async.series jobs, (err)-> if err? console.error err, "at end of jobs" - process.exit() else console.log "finished" - process.exit() done(err) From afb7fd6d16aa1eb534ce3dda2fbc21f182ba74c7 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Wed, 16 Sep 2015 16:27:47 +0100 Subject: [PATCH 9/9] remove extra logging in migration and change file paths in migration 2 --- migrations/2_doc_lines_delete_from_project.coffee | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/migrations/2_doc_lines_delete_from_project.coffee b/migrations/2_doc_lines_delete_from_project.coffee index 96c8d5b334..1e6a13dbf7 100644 --- a/migrations/2_doc_lines_delete_from_project.coffee +++ b/migrations/2_doc_lines_delete_from_project.coffee @@ -2,15 +2,14 @@ Settings = require "settings-sharelatex" fs = require("fs") mongojs = require("mongojs") ObjectId = mongojs.ObjectId -console.log Settings.mongo.url db = mongojs(Settings.mongo.url, ['projects', 'docs']) _ = require("lodash") async = require("async") exec = require("child_process").exec -finished_projects_path = "./finished-projects" -all_projects_path = "./all-projects" -unmigrated_docs_path = "./unmigrated" +finished_projects_path = "/tmp/finished-projects-2" +all_projects_path = "/tmp/all-projects-2" +unmigrated_docs_path = "/tmp/unmigrated-2" printProgress = -> @@ -108,7 +107,6 @@ isDocInDocCollection = (doc, callback)-> return callback(null, true) db.docs.find({_id: ObjectId(doc._id+"")}, {_id: 1}).limit 1, (err, foundDocs)-> exists = foundDocs.length > 0 - console.log doc._id, "Exits = "+exists callback err, exists getWhichDocsCanBeDeleted = (docs, callback = (err, docsToBeDeleted, unmigratedDocs)->)-> @@ -131,8 +129,6 @@ whipeDocLines = (project_id, mongoPath, callback)-> $unset: {} update.$unset["#{mongoPath}.lines"] = "" update.$unset["#{mongoPath}.rev"] = "" - console.log "project id = #{ObjectId(project_id+'')}", update - #callback() db.projects.update _id: ObjectId(project_id+''), update, callback