diff --git a/.gitignore b/.gitignore index b2cbe4ce63..abf5d3ff2f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ tags chat spelling real-time +migrations/*.js data tmp diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..bc8167022b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,11 @@ +v0.1.2 +------ + +* Re-brand open-source ShareLaTeX code as 'ShareLaTeX Community Edition'. +* The Dropbox and template code has been extracted out into a separate module and removed from the ShareLaTeX Community Edition. There should be no broken features due to lack of open source components now. +* Websockets and real-time data now go to a separate light-weight [real-time](https://github.com/sharelatex/real-time) service. +* Updated PDF viewer that loads page-by-page for much quicker loading times on large documents. +* Links are clickable in chat messages. +* Mathjax libraries are now served locally. +* Optimisation of Angular digest loop in editor should reduce CPU usage in certain cases. +* Numerous small bug fixes. diff --git a/Gruntfile.coffee b/Gruntfile.coffee index b330ed7dd4..ab591a76ab 100644 --- a/Gruntfile.coffee +++ b/Gruntfile.coffee @@ -7,6 +7,8 @@ semver = require "semver" knox = require "knox" crypto = require "crypto" async = require "async" +settings = require("settings-sharelatex") + SERVICES = [{ name: "web" @@ -56,6 +58,7 @@ module.exports = (grunt) -> grunt.loadNpmTasks 'grunt-available-tasks' grunt.loadNpmTasks 'grunt-concurrent' grunt.loadNpmTasks "grunt-contrib-coffee" + grunt.loadNpmTasks "grunt-shell" execute = {} @@ -83,7 +86,9 @@ module.exports = (grunt) -> options: bare:true - + shell: + migrate: + command: "./node_modules/east/bin/east migrate --adapter east-mongo --url #{settings.mongo.url}" availabletasks: tasks: @@ -162,7 +167,7 @@ module.exports = (grunt) -> Helpers.buildUpstartScripts() - grunt.registerTask 'migrate', 'run migrations', ['coffee:migrate'] + #grunt.registerTask 'migrate', "compile migrations and run them", ['coffee:migrate', 'shell:migrate'] Helpers = diff --git a/README.md b/README.md index 5592c47b73..816cf4958f 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,11 @@ We have detailed installation instructions in our wiki: **If you have any problems, have a look at our page of [Frequent Problems and Questions](https://github.com/sharelatex/sharelatex/wiki/FAQ).** +Upgrading +--------- + +If you are upgrading from a previous version of ShareLaTeX, please see the [Release Notes section on the Wiki] (https://github.com/sharelatex/sharelatex/wiki/Home) for all of the versions between your current version and the version you are upgrading to. + Dependencies ------------ diff --git a/config/settings.development.coffee.example b/config/settings.development.coffee.example index c51e2be595..857c8ce2ad 100644 --- a/config/settings.development.coffee.example +++ b/config/settings.development.coffee.example @@ -93,6 +93,14 @@ module.exports = # when emails are sent out and in generated links: siteUrl : 'http://localhost:3000' + # The websocket layer of ShareLaTeX runs as separate service. + # When running locally or in development, you can point the client to this + # service directly on port 3026. If you are running behind a reverse proxy (Nginx, etc) + # then websocketsUrl should be the same as siteUrl, with your reverse + # proxy responible for sending websocket traffic to the websocket service + # rather than connecting directly. + websocketsUrl: 'http://localhost:3026' + # If provided, a sessionSecret is used to sign cookies so that they cannot be # spoofed. This is recommended. security: diff --git a/migrations/1422460849371-doc-lines.coffee b/migrations/1_doc_lines.coffee similarity index 58% rename from migrations/1422460849371-doc-lines.coffee rename to migrations/1_doc_lines.coffee index a97d2907ac..9589b67f4c 100644 --- a/migrations/1422460849371-doc-lines.coffee +++ b/migrations/1_doc_lines.coffee @@ -1,31 +1,32 @@ - +Settings = require "settings-sharelatex" fs = require("fs") mongojs = require("mongojs") ObjectId = mongojs.ObjectId -db = mongojs('sharelatex', ['projects', 'docs']) +console.log Settings.mongo.url +db = mongojs(Settings.mongo.url, ['projects', 'docs']) _ = require("lodash") async = require("async") exec = require("child_process").exec finished_projects_path = "/tmp/finished-projects" all_projects_path = "/tmp/all-projects" +project_too_large_path = "/tmp/large_projects" + printProgress = -> exec "wc #{finished_projects_path}", (error, results) -> - #console.log results setTimeout printProgress, 1000 * 30 checkIfFileHasBeenProccessed = (project_id, callback)-> exec "grep #{project_id} #{finished_projects_path}", (error, results) -> hasBeenProcessed = _.include(results, project_id) - #console.log hasBeenProcessed, project_id - callback(null, hasBeenProcessed) + callback(error, hasBeenProcessed) loadProjectIds = (callback)-> + console.log "loading project ids from #{all_projects_path}" fs.readFile all_projects_path, "utf-8", (err, data)-> - console.log data.length ids = data.split("\n") - console.log ids.length + console.log "loaded #{ids.length} project ids from #{all_projects_path}" callback err, ids getAndWriteProjectids = (callback)-> @@ -38,9 +39,14 @@ getAndWriteProjectids = (callback)-> fs.writeFile all_projects_path, fileData, -> callback(err, ids) +markProjectAsToLargeAndFinished = (project_id, callback)-> + console.log "#{project_id} too large" + markProjectAsProcessed project_id, (err)-> + fs.appendFile project_too_large_path, "#{project_id}\n", callback + getProjectIds = (callback)-> exists = fs.existsSync all_projects_path - if exists + if exists loadProjectIds callback else getAndWriteProjectids callback @@ -52,8 +58,11 @@ getAllDocs = (project_id, callback = (error, docs) ->) -> db.projects.findOne _id:ObjectId(project_id), (error, project) -> return callback(error) if error? if !project? - console.error("No such project: #{project_id}") - return callback("no such project #{project_id}") + console.log "no such project #{project_id}" + return callback() + size = require("../node_modules/mongojs/node_modules/mongodb/node_modules/bson/").BSONPure.BSON.calculateObjectSize(project) + if size > 12000000 #12mb + return markProjectAsToLargeAndFinished project_id, callback findAllDocsInProject project, (error, docs) -> return callback(error) if error? return callback null, docs @@ -82,17 +91,33 @@ _findAllDocsInFolder = (folder = {}) -> return docs insertDocIntoDocCollection = (project_id, doc_id, lines, oldRev, callback)-> + if !project_id? + return callback("no project id") + if !doc_id? + return callback() + if !lines? + lines = [""] update = {} - update["_id"] = ObjectId(doc_id) + update["_id"] = ObjectId(doc_id.toString()) update["lines"] = lines update["project_id"] = ObjectId(project_id) - update["rev"] = oldRev - db.docs.insert _id: ObjectId(doc_id), callback + update["rev"] = oldRev || 0 + db.docs.insert update, callback saveDocsIntoMongo = (project_id, docs, callback)-> jobs = _.map docs, (doc)-> (cb)-> - insertDocIntoDocCollection project_id, doc._id, project_id.lines, doc.rev, cb + if !doc? + console.error "null doc in project #{project_id}" #just skip it, not a big deal + return cb() + insertDocIntoDocCollection project_id, doc._id, doc.lines, doc.rev, (err)-> + if err?.code == 11000 #duplicate key, doc already in there so its not a problem. + err = undefined + if err? + console.log "error inserting doc into doc collection", err + cb(err) + + async.series jobs, callback @@ -101,33 +126,36 @@ processNext = (project_id, callback)-> if hasBeenProcessed console.log "#{project_id} already procssed, skipping" return callback() + console.log "#{project_id} processing" getAllDocs project_id, (err, docs)-> if err? console.error err, project_id, "could not get all docs" - return callback() - saveDocsIntoMongo project_id, docs, -> - if err? - console.error err, project_id, "could not save docs into mongo" - return callback() - markProjectAsProcessed project_id, -> - callback() + return callback(err) + else + saveDocsIntoMongo project_id, docs, (err)-> + if err? + console.error err, project_id, "could not save docs into mongo" + return callback(err) + markProjectAsProcessed project_id, (err)-> + setTimeout( + -> callback(err) + ,500) -getProjectIds (err, ids)-> - printProgress() - jobs = _.map ids, (id)-> - return (cb)-> - processNext(id, cb) - async.series jobs, (err)-> - if err? - console.error err, "at end of jobs" - else - console.log "finished" - process.exit() - -exports.up = (next)-> - next() - - -exports.down = (next)-> + +exports.migrate = (client, done = ->)-> + getProjectIds (err, ids)-> + printProgress() + jobs = _.map ids, (id)-> + return (cb)-> + processNext(id, cb) + async.series jobs, (err)-> + if err? + console.error err, "at end of jobs" + else + console.log "finished" + done(err) + + +exports.rollback = (next)-> next() diff --git a/migrations/about_migrations.md b/migrations/about_migrations.md new file mode 100644 index 0000000000..146cb9c047 --- /dev/null +++ b/migrations/about_migrations.md @@ -0,0 +1,2 @@ +* if migration is stopped mid way it will start at the beginging next time +* to see the run migrations do db.getCollection('_migrations').find() you can't do db._migrations.find() \ No newline at end of file diff --git a/package.json b/package.json index e3d083ab32..e26c61eb32 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,9 @@ "description": "An online collaborative LaTeX editor", "dependencies": { "async": "^0.9.0", + "east": "^0.2.3", + "east-mongo": "^0.1.2", + "grunt-shell": "^1.1.1", "lodash": "^3.0.0", "mongojs": "^0.18.1", "rimraf": "~2.2.6", diff --git a/package/upstart/sharelatex-real-time b/package/upstart/sharelatex-real-time new file mode 100644 index 0000000000..7d44f0c8e2 --- /dev/null +++ b/package/upstart/sharelatex-real-time @@ -0,0 +1,28 @@ +description "sharelatex-web" +author "ShareLaTeX " + +start on (local-filesystems and net-device-up IFACE!=lo) +stop on shutdown + +respawn + +limit nofile 8192 8192 + +pre-start script + mkdir -p /var/log/sharelatex +end script + +script + SERVICE=real-time + USER=sharelatex + GROUP=sharelatex + # You may need to replace this with an absolute + # path to Node.js if it's not in your system PATH. + NODE=node + SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee + LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux + + echo $$ > /var/run/sharelatex-$SERVICE.pid + cd /var/www/sharelatex/$SERVICE + exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1 +end script