Release 0.1.3

force an npm rebuild to work around issue #129, missing bcrypt bindings

.gitignore

@@ -11,6 +11,8 @@ docstore
 tags
 chat
 spelling
+real-time
+migrations/*.js
 
 data
 tmp

CHANGELOG.md

@@ -0,0 +1,21 @@
+v0.1.3
+------
+
+* Fix bug with large files being corrupted when downloaded.
+* Update Ace editor to lastest release.
+* Lots of added null checks in the front-end javascript.
+* Don't crash if 'unzip' program isn't present.
+* Allow track-changes history to be packed into compressed 'packs'. This must be done manually for now.
+* Escape any shell special characters in the CLSI root path.
+
+v0.1.2
+------
+
+* Re-brand open-source ShareLaTeX code as 'ShareLaTeX Community Edition'.
+* The Dropbox and template code has been extracted out into a separate module and removed from the ShareLaTeX Community Edition. There should be no broken features due to lack of open source components now.
+* Websockets and real-time data now go to a separate light-weight [real-time](https://github.com/sharelatex/real-time) service.
+* Updated PDF viewer that loads page-by-page for much quicker loading times on large documents.
+* Links are clickable in chat messages.
+* Mathjax libraries are now served locally.
+* Optimisation of Angular digest loop in editor should reduce CPU usage in certain cases.
+* Numerous small bug fixes.

Gruntfile.coffee

@@ -7,43 +7,49 @@ semver = require "semver"
 knox = require "knox"
 crypto = require "crypto"
 async = require "async"
+settings = require("settings-sharelatex")
+
 
 SERVICES = [{
 	name: "web"
 	repo: "https://github.com/sharelatex/web-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.3"
+}, {
+	name: "real-time"
+	repo: "https://github.com/sharelatex/real-time-sharelatex.git"
+	version: "v0.1.3"
 }, {
 	name: "document-updater"
 	repo: "https://github.com/sharelatex/document-updater-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.3"
 }, {
 	name: "clsi"
 	repo: "https://github.com/sharelatex/clsi-sharelatex.git"
-	version: "v0.1.1"
+	version: "v0.1.3"
 }, {
 	name: "filestore"
 	repo: "https://github.com/sharelatex/filestore-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.3"
 }, {
 	name: "track-changes"
 	repo: "https://github.com/sharelatex/track-changes-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.3"
 }, {
 	name: "docstore"
 	repo: "https://github.com/sharelatex/docstore-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.3"
 }, {
 	name: "chat"
 	repo: "https://github.com/sharelatex/chat-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.3"
 }, {
 	name: "tags"
 	repo: "https://github.com/sharelatex/tags-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.3"
 }, {
 	name: "spelling"
 	repo: "https://github.com/sharelatex/spelling-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.3"
 }]
 
 module.exports = (grunt) ->
@@ -51,6 +57,9 @@ module.exports = (grunt) ->
 	grunt.loadNpmTasks 'grunt-execute'
 	grunt.loadNpmTasks 'grunt-available-tasks'
 	grunt.loadNpmTasks 'grunt-concurrent'
+	grunt.loadNpmTasks "grunt-contrib-coffee"
+	grunt.loadNpmTasks "grunt-shell"
+
 
 	execute = {}
 	for service in SERVICES
@@ -66,6 +75,20 @@ module.exports = (grunt) ->
 				options:
 					limit: SERVICES.length
 					logConcurrentOutput: true
+		coffee:
+			migrate: 
+				expand: true,
+				flatten: false,
+				cwd: './',
+				src: ['./migrations/*.coffee'],
+				dest: './',
+				ext: '.js'
+				options:
+					bare:true
+
+		shell:
+			migrate:
+				command: "./node_modules/east/bin/east migrate --adapter east-mongo --url #{settings?.mongo?.url}"
 
 		availabletasks:
 			tasks:
@@ -143,24 +166,32 @@ module.exports = (grunt) ->
 	grunt.registerTask "build:upstart_scripts", "Create upstart scripts for each service", () ->
 		Helpers.buildUpstartScripts()
 
+
+	#grunt.registerTask 'migrate', "compile migrations and run them", ['coffee:migrate', 'shell:migrate']
+
+
 	Helpers =
 		installService: (service, callback = (error) ->) ->
 			Helpers.cloneGitRepo service, (error) ->
 				return callback(error) if error?
 				Helpers.installNpmModules service, (error) ->
 					return callback(error) if error?
-					Helpers.runGruntInstall service, (error) ->
+					Helpers.rebuildNpmModules service, (error) ->
 						return callback(error) if error?
-						callback()
+						Helpers.runGruntInstall service, (error) ->
+							return callback(error) if error?
+							callback()
 
 		updateService: (service, callback = (error) ->) ->
 			Helpers.updateGitRepo service, (error) ->
 				return callback(error) if error?
 				Helpers.installNpmModules service, (error) ->
 					return callback(error) if error?
-					Helpers.runGruntInstall service, (error) ->
+					Helpers.rebuildNpmModules service, (error) ->
 						return callback(error) if error?
-						callback()
+						Helpers.runGruntInstall service, (error) ->
+							return callback(error) if error?
+							callback()
 
 		cloneGitRepo: (service, callback = (error) ->) ->
 			repo_src = service.repo
@@ -203,13 +234,21 @@ module.exports = (grunt) ->
 							proc = spawn "git", ["push", "--tags"], cwd: dir, stdio: "inherit"
 							proc.on "close", () ->
 								callback()
-								
+
 		installNpmModules: (service, callback = (error) ->) ->
 			dir = service.name
 			proc = spawn "npm", ["install"], stdio: "inherit", cwd: dir
 			proc.on "close", () ->
 				callback()
-				
+
+		# work around for https://github.com/npm/npm/issues/5400
+		# where binary modules are not built due to bug in npm
+		rebuildNpmModules: (service, callback = (error) ->) ->
+			dir = service.name
+			proc = spawn "npm", ["rebuild"], stdio: "inherit", cwd: dir
+			proc.on "close", () ->
+				callback()
+
 		createDataDirs: (callback = (error) ->) ->
 			DIRS = [
 				"tmp/dumpFolder"

README.md

@@ -3,6 +3,8 @@ ShareLaTeX
 
 [ShareLaTeX](https://www.sharelatex.com) is an open-source online real-time collaborative LaTeX editor. We run a hosted version at http://www.sharelatex.com, but you can also run your own local version, and contribute to the development of ShareLaTeX.
 
+*[If you want help installing and maintaining ShareLaTeX at your university or workplace, we offer an officially supported version called ShareLaTeX Server Pro. It also comes with extra security and admin features. Click here to find out more!](https://www.sharelatex.com/university/onsite.html)*
+
 Installation
 ------------
 
@@ -13,7 +15,11 @@ We have detailed installation instructions in our wiki:
 
 **If you have any problems, have a look at our page of [Frequent Problems and Questions](https://github.com/sharelatex/sharelatex/wiki/FAQ).**
 
-*[Please help us make ShareLaTeX as easy to install as possible by answering our quick survey about your system and needs](https://sharelatex.typeform.com/to/PLNits)*
+Upgrading
+---------
+
+If you are upgrading from a previous version of ShareLaTeX, please see the [Release Notes section on the Wiki] (https://github.com/sharelatex/sharelatex/wiki/Home) for all of the versions between your current version and the version you are upgrading to.
+
 
 Dependencies
 ------------
@@ -25,6 +31,8 @@ ShareLaTeX should run on OS X and Linux. You need:
 * A local instance of [Redis](http://redis.io/topics/quickstart) (version 2.6.12 or later) and [MongoDB](http://docs.mongodb.org/manual/installation/) running on their standard ports.
 * [TeXLive](https://www.tug.org/texlive/) 2013 or later with the `latexmk` program installed.
 
+ShareLaTeX needs a minimum of 2gb of memory, it is likely to be more than that though depending on usage.
+
 Other repositories
 ------------------
 
@@ -77,6 +85,10 @@ The backend API for managing project tags (folders).
 
 An API for running server-side spelling checking on ShareLaTeX documents.
 
+Dropbox
+-------
+
+Please note that certain features like Dropbox integration are not functional in the open source code base yet, despite appearing in the user interface. We're working on this, sorry!
 
 Contributing
 ------------

config/settings.development.coffee.example

@@ -93,6 +93,14 @@ module.exports =
 	# when emails are sent out and in generated links:
 	siteUrl : 'http://localhost:3000'
 	
+	# The websocket layer of ShareLaTeX runs as separate service.
+	# When running locally or in development, you can point the client to this
+	# service directly on port 3026. If you are running behind a reverse proxy (Nginx, etc)
+	# then websocketsUrl should be the same as siteUrl, with your reverse
+	# proxy responible for sending websocket traffic to the websocket service
+	# rather than connecting directly.
+	websocketsUrl: 'http://localhost:3026'
+	
 	# If provided, a sessionSecret is used to sign cookies so that they cannot be
 	# spoofed. This is recommended.
 	security:

migrations/1_doc_lines.coffee

@@ -0,0 +1,161 @@
+Settings = require "settings-sharelatex"
+fs = require("fs")
+mongojs = require("mongojs")
+ObjectId = mongojs.ObjectId
+console.log Settings.mongo.url
+db = mongojs(Settings.mongo.url, ['projects', 'docs'])
+_ = require("lodash")
+async = require("async")
+exec = require("child_process").exec
+
+finished_projects_path = "/tmp/finished-projects"
+all_projects_path = "/tmp/all-projects"
+project_too_large_path = "/tmp/large_projects"
+
+
+printProgress = ->
+	exec "wc #{finished_projects_path}", (error, results) ->
+		setTimeout printProgress, 1000 * 30
+
+checkIfFileHasBeenProccessed = (project_id, callback)->
+	exec "grep #{project_id} #{finished_projects_path}", (error, results) ->
+		hasBeenProcessed = _.include(results, project_id)
+		callback(error, hasBeenProcessed)
+
+loadProjectIds = (callback)->
+	console.log "loading project ids from #{all_projects_path}"
+	fs.readFile all_projects_path, "utf-8", (err, data)->
+		ids = data.split("\n")
+		console.log "loaded #{ids.length} project ids from #{all_projects_path}"
+		callback err, ids
+
+getAndWriteProjectids = (callback)->
+	console.log "finding all project id's - #{new Date().toString()}"
+	db.projects.find {}, {_id:1}, (err, ids)->
+		console.log "total found projects in mongo #{ids.length} - #{new Date().toString()}"
+		ids = _.pluck ids, '_id'
+		ids = _.filter ids, (id)-> id?
+		fileData = ids.join("\n")
+		fs.writeFile all_projects_path, fileData, ->
+			callback(err, ids)
+
+markProjectAsToLargeAndFinished = (project_id, callback)->
+	console.log "#{project_id} too large"
+	markProjectAsProcessed project_id, (err)->
+		fs.appendFile project_too_large_path, "#{project_id}\n", callback
+
+getProjectIds = (callback)->
+	exists = fs.existsSync all_projects_path
+	if exists
+		loadProjectIds callback
+	else
+		getAndWriteProjectids callback
+
+markProjectAsProcessed = (project_id, callback)->
+	fs.appendFile finished_projects_path, "#{project_id}\n", callback
+
+getAllDocs = (project_id, callback = (error, docs) ->) ->
+	db.projects.findOne _id:ObjectId(project_id), (error, project) ->
+		return callback(error) if error?
+		if !project?
+			console.log "no such project #{project_id}"
+			return callback()
+		size = require("../node_modules/mongojs/node_modules/mongodb/node_modules/bson/").BSONPure.BSON.calculateObjectSize(project)
+		if size > 12000000 #12mb
+			return markProjectAsToLargeAndFinished project_id, callback
+		findAllDocsInProject project, (error, docs) ->
+			return callback(error) if error?
+			return callback null, docs
+
+findAllDocsInProject = (project, callback = (error, docs) ->) ->
+	callback null, _findAllDocsInFolder project.rootFolder[0]
+
+_findDocInFolder = (folder = {}, doc_id, currentPath) ->
+	for doc, i in folder.docs or []
+		if doc?._id? and doc._id.toString() == doc_id.toString()
+			return {
+				doc: doc
+				mongoPath: "#{currentPath}.docs.#{i}"
+			}
+
+	for childFolder, i in folder.folders or []
+		result = _findDocInFolder childFolder, doc_id, "#{currentPath}.folders.#{i}"
+		return result if result?
+
+	return null
+
+_findAllDocsInFolder = (folder = {}) ->
+	docs = folder.docs or []
+	for childFolder in folder.folders or []
+		docs = docs.concat _findAllDocsInFolder childFolder
+	return docs
+
+insertDocIntoDocCollection = (project_id, doc_id, lines, oldRev, callback)->
+	if !project_id?
+		return callback("no project id")
+	if !doc_id?
+		return callback()
+	if !lines?
+		lines = [""]
+	update = {}
+	update["_id"] = ObjectId(doc_id.toString())
+	update["lines"] = lines
+	update["project_id"] = ObjectId(project_id)
+	update["rev"] = oldRev || 0
+	db.docs.insert update, callback
+
+saveDocsIntoMongo = (project_id, docs, callback)->
+	jobs = _.map docs, (doc)->
+		(cb)->
+			if !doc?
+				console.error "null doc in project #{project_id}" #just skip it, not a big deal
+				return cb()
+			insertDocIntoDocCollection project_id, doc._id, doc.lines, doc.rev, (err)->
+				if err?.code == 11000 #duplicate key, doc already in there so its not a problem.
+					err = undefined
+				if err?
+					console.log "error inserting doc into doc collection", err
+				cb(err)
+
+
+	async.series jobs, callback
+
+
+processNext = (project_id, callback)->
+	checkIfFileHasBeenProccessed project_id, (err, hasBeenProcessed)->
+		if hasBeenProcessed
+			console.log "#{project_id} already procssed, skipping"
+			return callback()
+		console.log "#{project_id} processing"
+		getAllDocs project_id, (err, docs)->
+			if err?
+				console.error err, project_id, "could not get all docs"
+				return callback(err)
+			else
+				saveDocsIntoMongo project_id, docs, (err)->
+					if err?
+						console.error err, project_id, "could not save docs into mongo"
+						return callback(err)
+					markProjectAsProcessed project_id, (err)->
+						setTimeout(
+							-> callback(err)
+						,500)
+
+
+
+exports.migrate = (client, done = ->)->
+	getProjectIds (err, ids)->
+		printProgress()
+		jobs = _.map ids, (id)->
+			return (cb)->
+				processNext(id, cb)
+		async.series jobs, (err)->
+			if err?
+				console.error err, "at end of jobs"
+			else
+				console.log "finished"
+			done(err)
+
+
+exports.rollback = (next)->
+	next()

migrations/about_migrations.md

@@ -0,0 +1,2 @@
+* if migration is stopped mid way it will start at the beginging next time
+* to see the run migrations do db.getCollection('_migrations').find() you can't do db._migrations.find()

package.json

@@ -1,11 +1,17 @@
 {
   "name": "sharelatex",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "An online collaborative LaTeX editor",
   "dependencies": {
     "async": "^0.9.0",
+    "east": "^0.2.3",
+    "east-mongo": "^0.1.2",
+    "grunt-shell": "^1.1.1",
+    "lodash": "^3.0.0",
+    "mongojs": "^0.18.1",
     "rimraf": "~2.2.6",
-    "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git"
+    "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git",
+    "underscore": "^1.7.0"
   },
   "devDependencies": {
     "grunt": "~0.4.2",
@@ -14,6 +20,7 @@
     "grunt-execute": "~0.1.5",
     "grunt-available-tasks": "~0.4.1",
     "grunt-concurrent": "~0.4.3",
+    "grunt-contrib-coffee": "~0.10.1",
     "semver": "~2.2.1",
     "knox": "~0.8.9"
   }

package/upstart/sharelatex-chat

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-clsi

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-docstore

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-document-updater

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-filestore

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-real-time

@@ -0,0 +1,28 @@
+description "sharelatex-web"
+author      "ShareLaTeX <team@sharelatex.com>"
+
+start on (local-filesystems and net-device-up IFACE!=lo)
+stop on shutdown
+
+respawn
+
+limit nofile 8192 8192
+
+pre-start script
+    mkdir -p /var/log/sharelatex
+end script
+
+script
+    SERVICE=real-time
+    USER=sharelatex
+    GROUP=sharelatex
+    # You may need to replace this with an absolute 
+    # path to Node.js if it's not in your system PATH.
+    NODE=node
+    SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee
+    LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
+
+    echo $$ > /var/run/sharelatex-$SERVICE.pid
+    cd /var/www/sharelatex/$SERVICE
+    exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
+end script

package/upstart/sharelatex-spelling

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-tags

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-template

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-track-changes

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-web

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script