From 0d97da15c4f94b297c08e973d88dd9eb80c67961 Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 10 Feb 2015 16:44:07 +0000 Subject: [PATCH 1/9] Create CHANGELOG --- CHANGELOG | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 CHANGELOG diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 0000000000..2b5fe5d866 --- /dev/null +++ b/CHANGELOG @@ -0,0 +1,10 @@ +v0.1.2 +------ + +* Re-brand open-source ShareLaTeX code as 'ShareLaTeX Community Edition'. +* The Dropbox and template code has been extracted out into a separate module and removed from the ShareLaTeX Community Edition. There should be no broken features due to lack of open source components now. +* Websockets and real-time data now go to a separate light-weight [real-time](https://github.com/sharelatex/real-time) service. +* Updated PDF viewer that loads page-by-page for much quicker loading times on large documents. +* Links are clickable in chat messages. +* Mathjax libraries are now served locally. +* Numerous small bug fixes. From a25333f8537b90b4e92245cd45eb9cd9a40251d3 Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 10 Feb 2015 16:44:21 +0000 Subject: [PATCH 2/9] Rename CHANGELOG to CHANGELOG.md --- CHANGELOG => CHANGELOG.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename CHANGELOG => CHANGELOG.md (100%) diff --git a/CHANGELOG b/CHANGELOG.md similarity index 100% rename from CHANGELOG rename to CHANGELOG.md From 9c894d49aa1d5a738ff42fadf5e5d34c2e60fb3e Mon Sep 17 00:00:00 2001 From: James Allen Date: Tue, 10 Feb 2015 16:45:30 +0000 Subject: [PATCH 3/9] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b5fe5d866..bc8167022b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,4 +7,5 @@ v0.1.2 * Updated PDF viewer that loads page-by-page for much quicker loading times on large documents. * Links are clickable in chat messages. * Mathjax libraries are now served locally. +* Optimisation of Angular digest loop in editor should reduce CPU usage in certain cases. * Numerous small bug fixes. From 0395a6d01aac15fe7f437b46f02abca53d43a7b3 Mon Sep 17 00:00:00 2001 From: James Allen Date: Wed, 11 Feb 2015 16:26:06 +0000 Subject: [PATCH 4/9] Update README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 5592c47b73..816cf4958f 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,11 @@ We have detailed installation instructions in our wiki: **If you have any problems, have a look at our page of [Frequent Problems and Questions](https://github.com/sharelatex/sharelatex/wiki/FAQ).** +Upgrading +--------- + +If you are upgrading from a previous version of ShareLaTeX, please see the [Release Notes section on the Wiki] (https://github.com/sharelatex/sharelatex/wiki/Home) for all of the versions between your current version and the version you are upgrading to. + Dependencies ------------ From 630996166bae42de58ab45f3019f1d37cc91dc70 Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 16 Feb 2015 10:02:55 +0000 Subject: [PATCH 5/9] Create sharelatex-real-time --- package/upstart/sharelatex-real-time | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 package/upstart/sharelatex-real-time diff --git a/package/upstart/sharelatex-real-time b/package/upstart/sharelatex-real-time new file mode 100644 index 0000000000..7d44f0c8e2 --- /dev/null +++ b/package/upstart/sharelatex-real-time @@ -0,0 +1,28 @@ +description "sharelatex-web" +author "ShareLaTeX " + +start on (local-filesystems and net-device-up IFACE!=lo) +stop on shutdown + +respawn + +limit nofile 8192 8192 + +pre-start script + mkdir -p /var/log/sharelatex +end script + +script + SERVICE=real-time + USER=sharelatex + GROUP=sharelatex + # You may need to replace this with an absolute + # path to Node.js if it's not in your system PATH. + NODE=node + SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee + LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux + + echo $$ > /var/run/sharelatex-$SERVICE.pid + cd /var/www/sharelatex/$SERVICE + exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1 +end script From 8d3a57bd4b142d3edac1b83469d24b8d13fc8e3d Mon Sep 17 00:00:00 2001 From: James Allen Date: Mon, 16 Feb 2015 10:05:14 +0000 Subject: [PATCH 6/9] Update settings.development.coffee.example --- config/settings.development.coffee.example | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/config/settings.development.coffee.example b/config/settings.development.coffee.example index c51e2be595..857c8ce2ad 100644 --- a/config/settings.development.coffee.example +++ b/config/settings.development.coffee.example @@ -93,6 +93,14 @@ module.exports = # when emails are sent out and in generated links: siteUrl : 'http://localhost:3000' + # The websocket layer of ShareLaTeX runs as separate service. + # When running locally or in development, you can point the client to this + # service directly on port 3026. If you are running behind a reverse proxy (Nginx, etc) + # then websocketsUrl should be the same as siteUrl, with your reverse + # proxy responible for sending websocket traffic to the websocket service + # rather than connecting directly. + websocketsUrl: 'http://localhost:3026' + # If provided, a sessionSecret is used to sign cookies so that they cannot be # spoofed. This is recommended. security: From bfe4dd018be4a66dccf3313b2669c50badf06796 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Tue, 10 Feb 2015 17:25:13 +0000 Subject: [PATCH 7/9] added migrations with east package --- Gruntfile.coffee | 9 +- ...71-doc-lines.coffee => 1_doc_lines.coffee} | 85 ++++++++++++------- migrations/about_migrations.md | 2 + package.json | 3 + 4 files changed, 64 insertions(+), 35 deletions(-) rename migrations/{1422460849371-doc-lines.coffee => 1_doc_lines.coffee} (66%) create mode 100644 migrations/about_migrations.md diff --git a/Gruntfile.coffee b/Gruntfile.coffee index 3b3339e417..d8e5f5ee3c 100644 --- a/Gruntfile.coffee +++ b/Gruntfile.coffee @@ -7,6 +7,8 @@ semver = require "semver" knox = require "knox" crypto = require "crypto" async = require "async" +settings = require("settings-sharelatex") + SERVICES = [{ name: "web" @@ -56,6 +58,7 @@ module.exports = (grunt) -> grunt.loadNpmTasks 'grunt-available-tasks' grunt.loadNpmTasks 'grunt-concurrent' grunt.loadNpmTasks "grunt-contrib-coffee" + grunt.loadNpmTasks "grunt-shell" execute = {} @@ -83,7 +86,9 @@ module.exports = (grunt) -> options: bare:true - + shell: + migrate: + command: "./node_modules/east/bin/east migrate --adapter east-mongo --url #{settings.mongo.url}" availabletasks: tasks: @@ -162,7 +167,7 @@ module.exports = (grunt) -> Helpers.buildUpstartScripts() - grunt.registerTask 'migrate', 'run migrations', ['coffee:migrate'] + grunt.registerTask 'migrate', "compile migrations and run them", ['coffee:migrate', 'shell:migrate'] Helpers = diff --git a/migrations/1422460849371-doc-lines.coffee b/migrations/1_doc_lines.coffee similarity index 66% rename from migrations/1422460849371-doc-lines.coffee rename to migrations/1_doc_lines.coffee index a97d2907ac..9e0f8ae7f6 100644 --- a/migrations/1422460849371-doc-lines.coffee +++ b/migrations/1_doc_lines.coffee @@ -1,8 +1,8 @@ - +Settings = require "settings-sharelatex" fs = require("fs") mongojs = require("mongojs") ObjectId = mongojs.ObjectId -db = mongojs('sharelatex', ['projects', 'docs']) +db = mongojs(Settings.mongo.url, ['projects', 'docs']) _ = require("lodash") async = require("async") exec = require("child_process").exec @@ -19,13 +19,13 @@ checkIfFileHasBeenProccessed = (project_id, callback)-> exec "grep #{project_id} #{finished_projects_path}", (error, results) -> hasBeenProcessed = _.include(results, project_id) #console.log hasBeenProcessed, project_id - callback(null, hasBeenProcessed) + callback(error, hasBeenProcessed) loadProjectIds = (callback)-> + console.log "loading project ids from #{all_projects_path}" fs.readFile all_projects_path, "utf-8", (err, data)-> - console.log data.length ids = data.split("\n") - console.log ids.length + console.log "loaded #{ids.length} project ids from #{all_projects_path}" callback err, ids getAndWriteProjectids = (callback)-> @@ -52,7 +52,6 @@ getAllDocs = (project_id, callback = (error, docs) ->) -> db.projects.findOne _id:ObjectId(project_id), (error, project) -> return callback(error) if error? if !project? - console.error("No such project: #{project_id}") return callback("no such project #{project_id}") findAllDocsInProject project, (error, docs) -> return callback(error) if error? @@ -82,17 +81,34 @@ _findAllDocsInFolder = (folder = {}) -> return docs insertDocIntoDocCollection = (project_id, doc_id, lines, oldRev, callback)-> + if !project_id? + return callback("no project id") + if !doc_id? + return callback("no doc id. project=#{project_id}") + if !lines? + return callback("no lines") update = {} - update["_id"] = ObjectId(doc_id) + update["_id"] = ObjectId(doc_id.toString()) update["lines"] = lines update["project_id"] = ObjectId(project_id) - update["rev"] = oldRev - db.docs.insert _id: ObjectId(doc_id), callback + update["rev"] = oldRev || 0 + # console.log update + db.docs.insert update, callback saveDocsIntoMongo = (project_id, docs, callback)-> jobs = _.map docs, (doc)-> (cb)-> - insertDocIntoDocCollection project_id, doc._id, project_id.lines, doc.rev, cb + if !doc? + console.error "null doc in project #{project_id}" + return cb() + insertDocIntoDocCollection project_id, doc._id, doc.lines, doc.rev, (err)-> + if err?.code == 11000 #duplicate key, doc already in there so its not a problem. + err = undefined + if err? + console.log "error inserting doc into doc collection", err + cb(err) + + async.series jobs, callback @@ -101,33 +117,36 @@ processNext = (project_id, callback)-> if hasBeenProcessed console.log "#{project_id} already procssed, skipping" return callback() + console.log "#{project_id} processing" getAllDocs project_id, (err, docs)-> if err? console.error err, project_id, "could not get all docs" - return callback() - saveDocsIntoMongo project_id, docs, -> - if err? - console.error err, project_id, "could not save docs into mongo" - return callback() - markProjectAsProcessed project_id, -> - callback() + return callback(err) + else + saveDocsIntoMongo project_id, docs, (err)-> + if err? + console.error err, project_id, "could not save docs into mongo" + return callback(err) + markProjectAsProcessed project_id, (err)-> + setTimeout( + -> callback(err) + ,100) + -getProjectIds (err, ids)-> - printProgress() - jobs = _.map ids, (id)-> - return (cb)-> - processNext(id, cb) - async.series jobs, (err)-> - if err? - console.error err, "at end of jobs" - else - console.log "finished" - process.exit() -exports.up = (next)-> +exports.migrate = (client, done)-> + getProjectIds (err, ids)-> + printProgress() + jobs = _.map ids, (id)-> + return (cb)-> + processNext(id, cb) + async.series jobs, (err)-> + if err? + console.error err, "at end of jobs" + else + console.log "finished" + done(err) - next() - - -exports.down = (next)-> + +exports.rollback = (next)-> next() diff --git a/migrations/about_migrations.md b/migrations/about_migrations.md new file mode 100644 index 0000000000..146cb9c047 --- /dev/null +++ b/migrations/about_migrations.md @@ -0,0 +1,2 @@ +* if migration is stopped mid way it will start at the beginging next time +* to see the run migrations do db.getCollection('_migrations').find() you can't do db._migrations.find() \ No newline at end of file diff --git a/package.json b/package.json index a50146f3ef..cee7ffc776 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,9 @@ "description": "An online collaborative LaTeX editor", "dependencies": { "async": "^0.9.0", + "east": "^0.2.3", + "east-mongo": "^0.1.2", + "grunt-shell": "^1.1.1", "lodash": "^3.0.0", "mongojs": "^0.18.1", "rimraf": "~2.2.6", From 685abf8cbed98aa7b159ca0712e656b3ccbc8409 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Fri, 20 Feb 2015 15:40:11 +0000 Subject: [PATCH 8/9] finished migration script --- .gitignore | 1 + migrations/1_doc_lines.coffee | 33 +++++++++++++++++++++------------ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index b2cbe4ce63..abf5d3ff2f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ tags chat spelling real-time +migrations/*.js data tmp diff --git a/migrations/1_doc_lines.coffee b/migrations/1_doc_lines.coffee index 9e0f8ae7f6..9589b67f4c 100644 --- a/migrations/1_doc_lines.coffee +++ b/migrations/1_doc_lines.coffee @@ -2,6 +2,7 @@ Settings = require "settings-sharelatex" fs = require("fs") mongojs = require("mongojs") ObjectId = mongojs.ObjectId +console.log Settings.mongo.url db = mongojs(Settings.mongo.url, ['projects', 'docs']) _ = require("lodash") async = require("async") @@ -9,16 +10,16 @@ exec = require("child_process").exec finished_projects_path = "/tmp/finished-projects" all_projects_path = "/tmp/all-projects" +project_too_large_path = "/tmp/large_projects" + printProgress = -> exec "wc #{finished_projects_path}", (error, results) -> - #console.log results setTimeout printProgress, 1000 * 30 checkIfFileHasBeenProccessed = (project_id, callback)-> exec "grep #{project_id} #{finished_projects_path}", (error, results) -> hasBeenProcessed = _.include(results, project_id) - #console.log hasBeenProcessed, project_id callback(error, hasBeenProcessed) loadProjectIds = (callback)-> @@ -38,9 +39,14 @@ getAndWriteProjectids = (callback)-> fs.writeFile all_projects_path, fileData, -> callback(err, ids) +markProjectAsToLargeAndFinished = (project_id, callback)-> + console.log "#{project_id} too large" + markProjectAsProcessed project_id, (err)-> + fs.appendFile project_too_large_path, "#{project_id}\n", callback + getProjectIds = (callback)-> exists = fs.existsSync all_projects_path - if exists + if exists loadProjectIds callback else getAndWriteProjectids callback @@ -52,7 +58,11 @@ getAllDocs = (project_id, callback = (error, docs) ->) -> db.projects.findOne _id:ObjectId(project_id), (error, project) -> return callback(error) if error? if !project? - return callback("no such project #{project_id}") + console.log "no such project #{project_id}" + return callback() + size = require("../node_modules/mongojs/node_modules/mongodb/node_modules/bson/").BSONPure.BSON.calculateObjectSize(project) + if size > 12000000 #12mb + return markProjectAsToLargeAndFinished project_id, callback findAllDocsInProject project, (error, docs) -> return callback(error) if error? return callback null, docs @@ -84,27 +94,26 @@ insertDocIntoDocCollection = (project_id, doc_id, lines, oldRev, callback)-> if !project_id? return callback("no project id") if !doc_id? - return callback("no doc id. project=#{project_id}") + return callback() if !lines? - return callback("no lines") + lines = [""] update = {} update["_id"] = ObjectId(doc_id.toString()) update["lines"] = lines update["project_id"] = ObjectId(project_id) update["rev"] = oldRev || 0 - # console.log update db.docs.insert update, callback saveDocsIntoMongo = (project_id, docs, callback)-> jobs = _.map docs, (doc)-> (cb)-> if !doc? - console.error "null doc in project #{project_id}" + console.error "null doc in project #{project_id}" #just skip it, not a big deal return cb() insertDocIntoDocCollection project_id, doc._id, doc.lines, doc.rev, (err)-> if err?.code == 11000 #duplicate key, doc already in there so its not a problem. err = undefined - if err? + if err? console.log "error inserting doc into doc collection", err cb(err) @@ -130,11 +139,11 @@ processNext = (project_id, callback)-> markProjectAsProcessed project_id, (err)-> setTimeout( -> callback(err) - ,100) + ,500) - -exports.migrate = (client, done)-> + +exports.migrate = (client, done = ->)-> getProjectIds (err, ids)-> printProgress() jobs = _.map ids, (id)-> From 4d01a543918da117d94dcc134bf9d1eb5190b1f6 Mon Sep 17 00:00:00 2001 From: Henry Oswald Date: Fri, 20 Feb 2015 15:45:36 +0000 Subject: [PATCH 9/9] commented out the migrate command so people don't run it yet --- Gruntfile.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gruntfile.coffee b/Gruntfile.coffee index d8e5f5ee3c..df8cf03045 100644 --- a/Gruntfile.coffee +++ b/Gruntfile.coffee @@ -167,7 +167,7 @@ module.exports = (grunt) -> Helpers.buildUpstartScripts() - grunt.registerTask 'migrate', "compile migrations and run them", ['coffee:migrate', 'shell:migrate'] + #grunt.registerTask 'migrate', "compile migrations and run them", ['coffee:migrate', 'shell:migrate'] Helpers =