Release 0.1.4

Update NGINX Configuration

.gitignore

@@ -11,6 +11,8 @@ docstore
 tags
 chat
 spelling
+real-time
+migrations/*.js
 
 data
 tmp

CHANGELOG.md

@@ -0,0 +1,21 @@
+v0.1.3
+------
+
+* Fix bug with large files being corrupted when downloaded.
+* Update Ace editor to lastest release.
+* Lots of added null checks in the front-end javascript.
+* Don't crash if 'unzip' program isn't present.
+* Allow track-changes history to be packed into compressed 'packs'. This must be done manually for now.
+* Escape any shell special characters in the CLSI root path.
+
+v0.1.2
+------
+
+* Re-brand open-source ShareLaTeX code as 'ShareLaTeX Community Edition'.
+* The Dropbox and template code has been extracted out into a separate module and removed from the ShareLaTeX Community Edition. There should be no broken features due to lack of open source components now.
+* Websockets and real-time data now go to a separate light-weight [real-time](https://github.com/sharelatex/real-time) service.
+* Updated PDF viewer that loads page-by-page for much quicker loading times on large documents.
+* Links are clickable in chat messages.
+* Mathjax libraries are now served locally.
+* Optimisation of Angular digest loop in editor should reduce CPU usage in certain cases.
+* Numerous small bug fixes.

Gruntfile.coffee

@@ -7,43 +7,49 @@ semver = require "semver"
 knox = require "knox"
 crypto = require "crypto"
 async = require "async"
+settings = require("settings-sharelatex")
+
 
 SERVICES = [{
 	name: "web"
 	repo: "https://github.com/sharelatex/web-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.4"
+}, {
+	name: "real-time"
+	repo: "https://github.com/sharelatex/real-time-sharelatex.git"
+	version: "v0.1.4"
 }, {
 	name: "document-updater"
 	repo: "https://github.com/sharelatex/document-updater-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.4"
 }, {
 	name: "clsi"
 	repo: "https://github.com/sharelatex/clsi-sharelatex.git"
-	version: "v0.1.1"
+	version: "v0.1.4"
 }, {
 	name: "filestore"
 	repo: "https://github.com/sharelatex/filestore-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.4"
 }, {
 	name: "track-changes"
 	repo: "https://github.com/sharelatex/track-changes-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.4"
 }, {
 	name: "docstore"
 	repo: "https://github.com/sharelatex/docstore-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.4"
 }, {
 	name: "chat"
 	repo: "https://github.com/sharelatex/chat-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.4"
 }, {
 	name: "tags"
 	repo: "https://github.com/sharelatex/tags-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.4"
 }, {
 	name: "spelling"
 	repo: "https://github.com/sharelatex/spelling-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.4"
 }]
 
 module.exports = (grunt) ->
@@ -51,6 +57,9 @@ module.exports = (grunt) ->
 	grunt.loadNpmTasks 'grunt-execute'
 	grunt.loadNpmTasks 'grunt-available-tasks'
 	grunt.loadNpmTasks 'grunt-concurrent'
+	grunt.loadNpmTasks "grunt-contrib-coffee"
+	grunt.loadNpmTasks "grunt-shell"
+
 
 	execute = {}
 	for service in SERVICES
@@ -66,6 +75,20 @@ module.exports = (grunt) ->
 				options:
 					limit: SERVICES.length
 					logConcurrentOutput: true
+		coffee:
+			migrate: 
+				expand: true,
+				flatten: false,
+				cwd: './',
+				src: ['./migrations/*.coffee'],
+				dest: './',
+				ext: '.js'
+				options:
+					bare:true
+
+		shell:
+			migrate:
+				command: "./node_modules/east/bin/east migrate --adapter east-mongo --url #{settings?.mongo?.url}"
 
 		availabletasks:
 			tasks:
@@ -138,29 +161,35 @@ module.exports = (grunt) ->
 		Helpers.checkMake @async()
 	grunt.registerTask "check", "Check that you have the required dependencies installed", ["check:redis", "check:latexmk", "check:s3", "check:fs", "check:aspell"]
 
-	grunt.registerTask "build:deb", "Build an installable .deb file from the current directory", () ->
-		Helpers.buildDeb @async()
 	grunt.registerTask "build:upstart_scripts", "Create upstart scripts for each service", () ->
 		Helpers.buildUpstartScripts()
 
+
+	#grunt.registerTask 'migrate', "compile migrations and run them", ['coffee:migrate', 'shell:migrate']
+
+
 	Helpers =
 		installService: (service, callback = (error) ->) ->
 			Helpers.cloneGitRepo service, (error) ->
 				return callback(error) if error?
 				Helpers.installNpmModules service, (error) ->
 					return callback(error) if error?
-					Helpers.runGruntInstall service, (error) ->
+					Helpers.rebuildNpmModules service, (error) ->
 						return callback(error) if error?
-						callback()
+						Helpers.runGruntInstall service, (error) ->
+							return callback(error) if error?
+							callback()
 
 		updateService: (service, callback = (error) ->) ->
 			Helpers.updateGitRepo service, (error) ->
 				return callback(error) if error?
 				Helpers.installNpmModules service, (error) ->
 					return callback(error) if error?
-					Helpers.runGruntInstall service, (error) ->
+					Helpers.rebuildNpmModules service, (error) ->
 						return callback(error) if error?
-						callback()
+						Helpers.runGruntInstall service, (error) ->
+							return callback(error) if error?
+							callback()
 
 		cloneGitRepo: (service, callback = (error) ->) ->
 			repo_src = service.repo
@@ -210,6 +239,14 @@ module.exports = (grunt) ->
 			proc.on "close", () ->
 				callback()
 
+		# work around for https://github.com/npm/npm/issues/5400
+		# where binary modules are not built due to bug in npm
+		rebuildNpmModules: (service, callback = (error) ->) ->
+			dir = service.name
+			proc = spawn "npm", ["rebuild"], stdio: "inherit", cwd: dir
+			proc.on "close", () ->
+				callback()
+
 		createDataDirs: (callback = (error) ->) ->
 			DIRS = [
 				"tmp/dumpFolder"
@@ -418,93 +455,7 @@ module.exports = (grunt) ->
 					return callback()
 
 		buildUpstartScripts: () ->
-			template = fs.readFileSync("package/upstart/sharelatex-template").toString()
+			template = fs.readFileSync("package/upstart/sharelatex-template.conf").toString()
 			for service in SERVICES
-				fs.writeFileSync "package/upstart/sharelatex-#{service.name}", template.replace(/__SERVICE__/g, service.name)
-
-		buildPackageSettingsFile: () ->
-			config = fs.readFileSync("config/settings.development.coffee.example").toString()
-			config = config.replace /DATA_DIR.*/, "DATA_DIR = '/var/lib/sharelatex/data'"
-			config = config.replace /TMP_DIR.*/, "TMP_DIR = '/var/lib/sharelatex/tmp'"
-			fs.writeFileSync "package/config/settings.coffee", config
-
-		buildDeb: (callback = (error) ->) ->
-			command = ["-s", "dir", "-t", "deb", "-n", "sharelatex", "-v", "0.0.1", "--verbose"]
-			command.push(
-				"--maintainer", "ShareLaTeX <team@sharelatex.com>"
-				"--config-files", "/etc/sharelatex/settings.coffee"
-				"--config-files", "/etc/nginx/conf.d/sharelatex.conf"
-				"--directories",  "/var/lib/sharelatex"
-				"--directories",  "/var/log/sharelatex"
-			)
-
-			command.push(
-				"--depends", "redis-server > 2.6.12"
-				"--depends", "mongodb-org > 2.4.0"
-				"--depends", "nodejs > 0.10.0"
-			)
-			
-			@buildPackageSettingsFile()
-
-			@buildUpstartScripts()
-			for service in SERVICES
-				command.push(
-					"--deb-upstart", "package/upstart/sharelatex-#{service.name}"
-				)
-
-			after_install_script = """
-				#!/bin/sh
-				# Create random secret keys (twice, once for http auth pass, once for cookie secret).
-				sed -i "0,/CRYPTO_RANDOM/s/CRYPTO_RANDOM/$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 64 | head -n 1)/" /etc/sharelatex/settings.coffee
-				sed -i "0,/CRYPTO_RANDOM/s/CRYPTO_RANDOM/$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 64 | head -n 1)/" /etc/sharelatex/settings.coffee
-				
-				sudo adduser --system --group --home /var/www/sharelatex --no-create-home sharelatex
-
-				mkdir -p /var/log/sharelatex
-				chown sharelatex:sharelatex /var/log/sharelatex
-				
-				mkdir -p /var/lib/sharelatex
-
-			"""
-
-			for dir in ["data/user_files", "tmp/uploads", "data/compiles", "data/cache", "tmp/dumpFolder"]
-				after_install_script += """
-					mkdir -p /var/lib/sharelatex/#{dir}
-					
-				"""
-			
-			after_install_script += """
-				chown -R sharelatex:sharelatex /var/lib/sharelatex
-				
-			"""	
-
-			for service in SERVICES
-				after_install_script += "service sharelatex-#{service.name} restart\n"
-			fs.writeFileSync "package/scripts/after_install.sh", after_install_script
-			command.push("--after-install", "package/scripts/after_install.sh")
-
-			command.push("--exclude", "**/.git")
-			command.push("--exclude", "**/node_modules/grunt-*")
-			for path in ["filestore/user_files", "filestore/uploads", "clsi/cache", "clsi/compiles"]
-				command.push "--exclude", path
-
-			for service in SERVICES
-				command.push "#{service.name}=/var/www/sharelatex/"
-
-			command.push(
-				"package/config/settings.coffee=/etc/sharelatex/settings.coffee"
-				"package/nginx/sharelatex=/etc/nginx/conf.d/sharelatex.conf"
-			)
-			console.log "fpm " + command.join(" ")
-			proc = spawn "fpm", command, stdio: "inherit"
-			proc.on "close", (code) ->
-				if code != 0
-					callback(new Error("exit code: #{code}"))
-				else
-					callback()
-
-			
-
-
-
+				fs.writeFileSync "package/upstart/sharelatex-#{service.name}.conf", template.replace(/__SERVICE__/g, service.name)
 

README.md

@@ -3,6 +3,8 @@ ShareLaTeX
 
 [ShareLaTeX](https://www.sharelatex.com) is an open-source online real-time collaborative LaTeX editor. We run a hosted version at http://www.sharelatex.com, but you can also run your own local version, and contribute to the development of ShareLaTeX.
 
+*[If you want help installing and maintaining ShareLaTeX at your university or workplace, we offer an officially supported version called ShareLaTeX Server Pro. It also comes with extra security and admin features. Click here to find out more!](https://www.sharelatex.com/university/onsite.html)*
+
 Installation
 ------------
 
@@ -13,18 +15,24 @@ We have detailed installation instructions in our wiki:
 
 **If you have any problems, have a look at our page of [Frequent Problems and Questions](https://github.com/sharelatex/sharelatex/wiki/FAQ).**
 
-*[Please help us make ShareLaTeX as easy to install as possible by answering our quick survey about your system and needs](https://sharelatex.typeform.com/to/PLNits)*
+Upgrading
+---------
+
+If you are upgrading from a previous version of ShareLaTeX, please see the [Release Notes section on the Wiki] (https://github.com/sharelatex/sharelatex/wiki/Home) for all of the versions between your current version and the version you are upgrading to.
+
 
 Dependencies
 ------------
 
 ShareLaTeX should run on OS X and Linux. You need:
 
-* [Node.js](http://nodejs.org/) 0.10 or greater. We recommend that you use [nvm](https://github.com/creationix/nvm) to install it.
+* [Node.js](http://nodejs.org/) 0.10.x. We recommend that you use [nvm](https://github.com/creationix/nvm) to install it.
 * The [grunt](http://gruntjs.com/) command line tools (Run `npm install -g grunt-cli` to install them)
 * A local instance of [Redis](http://redis.io/topics/quickstart) (version 2.6.12 or later) and [MongoDB](http://docs.mongodb.org/manual/installation/) running on their standard ports.
 * [TeXLive](https://www.tug.org/texlive/) 2013 or later with the `latexmk` program installed.
 
+ShareLaTeX needs a minimum of 2gb of memory, it is likely to be more than that though depending on usage.
+
 Other repositories
 ------------------
 
@@ -77,6 +85,10 @@ The backend API for managing project tags (folders).
 
 An API for running server-side spelling checking on ShareLaTeX documents.
 
+Dropbox
+-------
+
+Please note that certain features like Dropbox integration are not functional in the open source code base yet, despite appearing in the user interface. We're working on this, sorry!
 
 Contributing
 ------------

config/settings.development.coffee.example

@@ -63,13 +63,13 @@ module.exports =
 	# To use Amazon S3 as a storage backend, comment out the above config, and
 	# uncomment the following, filling in your key, secret, and bucket name:
 	#
-	# filestore:
-	# 	backend: "s3"
-	# 	stores:
-	# 		user_files: "BUCKET_NAME"
-	# 	s3:
-	# 		key: "AWS_KEY"
-	# 		secret: "AWS_SECRET"
+	#filestore:
+	#	backend: "s3"
+	#	stores:
+	#		user_files: "BUCKET_NAME"
+	#	s3:
+	#		key: "AWS_KEY"
+	#		secret: "AWS_SECRET"
 	# 		
 
 	# Local disk caching
@@ -93,6 +93,14 @@ module.exports =
 	# when emails are sent out and in generated links:
 	siteUrl : 'http://localhost:3000'
 	
+	# The websocket layer of ShareLaTeX runs as separate service.
+	# When running locally or in development, you can point the client to this
+	# service directly on port 3026. If you are running behind a reverse proxy (Nginx, etc)
+	# then websocketsUrl should be the same as siteUrl, with your reverse
+	# proxy responible for sending websocket traffic to the websocket service
+	# rather than connecting directly.
+	websocketsUrl: 'http://localhost:3026'
+	
 	# If provided, a sessionSecret is used to sign cookies so that they cannot be
 	# spoofed. This is recommended.
 	security:
@@ -156,34 +164,37 @@ module.exports =
 	# an HTTP API running on a different port. Generally you
 	# can leave these as they are unless you have some other services
 	# running which conflict, or want to run the web process on port 80.
-	# internal:
-	# 	web:
-	# 		port: webPort = 3000
-	# 		host: "localhost"
-	# 	documentupdater:
-	# 		port: docUpdaterPort = 3003
-	# 		host: "localhost"
-	# 	filestore:
-	# 		port: filestorePort = 3009
-	# 		host: "localhost"
-	# 	chat:
-	# 		port: chatPort = 3010
-	# 		host: "localhost"
-	# 	tags:
-	# 		port: tagsPort = 3012
-	# 		host: "localhost"
-	# 	clsi:
-	# 		port: clsiPort = 3013
-	# 		host: "localhost"
-	# 	trackchanges:
-	# 		port: trackchangesPort = 3015
-	# 		host: "localhost"
-	# 	docstore:
-	# 		port: docstorePort = 3016
-	# 		host: "localhost"
-	# 	spelling:
+	#internal:
+	#	web:
+	#		port: webPort = 3000
+	#		host: "localhost"
+	#	documentupdater:
+	#		port: docUpdaterPort = 3003
+	#		host: "localhost"
+	#	filestore:
+	#		port: filestorePort = 3009
+	#		host: "localhost"
+	#	chat:
+	#		port: chatPort = 3010
+	#		host: "localhost"
+	#	tags:
+	#		port: tagsPort = 3012
+	#		host: "localhost"
+	#	clsi:
+	#		port: clsiPort = 3013
+	#		host: "localhost"
+	#	trackchanges:
+	#		port: trackchangesPort = 3015
+	#		host: "localhost"
+	#	docstore:
+	#		port: docstorePort = 3016
+	#		host: "localhost"
+	#	spelling:
 	# 		port: spellingPort = 3005
-	# 		host: "localhost"
+	#		host: "localhost"
+	#	realTime:
+	#		port: realTimeport = 3026
+	#		host: "localhost"
 
 	# If you change the above config, or run some services on remote servers,
 	# you need to tell the other services where to find them:
@@ -192,22 +203,22 @@ module.exports =
 			url: "http://localhost:3000"
 			user: httpAuthUser
 			pass: httpAuthPass
-	# 	documentupdater:
-	# 		url : "http://localhost:#{docUpdaterPort}"
-	# 	clsi:
-	# 		url: "http://localhost:#{clsiPort}"
-	# 	filestore:
-	# 		url: "http://localhost:#{filestorePort}"
-	# 	trackchanges:
-	# 		url: "http://localhost:#{trackchangesPort}"
-	# 	docstore:
-	# 		url: "http://localhost:#{docstorePort}"
-	# 	tags:
-	# 		url: "http://localhost:#{tagsPort}"
-	# 	spelling:
-	# 		url: "http://localhost:#{spellingPort}"
-	# 	chat:
-	# 		url: "http://localhost:#{chatPort}"
+	#	documentupdater:
+	#		url : "http://localhost:#{docUpdaterPort}"
+	#	clsi:
+	#		url: "http://localhost:#{clsiPort}"
+	#	filestore:
+	#		url: "http://localhost:#{filestorePort}"
+	#	trackchanges:
+	#		url: "http://localhost:#{trackchangesPort}"
+	#	docstore:
+	#		url: "http://localhost:#{docstorePort}"
+	#	tags:
+	#		url: "http://localhost:#{tagsPort}"
+	#	spelling:
+	#		url: "http://localhost:#{spellingPort}"
+	#	chat:
+	#		url: "http://localhost:#{chatPort}"
 	
 
 # With lots of incoming and outgoing HTTP connections to different services,

migrations/1_doc_lines.coffee

@@ -0,0 +1,161 @@
+Settings = require "settings-sharelatex"
+fs = require("fs")
+mongojs = require("mongojs")
+ObjectId = mongojs.ObjectId
+console.log Settings.mongo.url
+db = mongojs(Settings.mongo.url, ['projects', 'docs'])
+_ = require("lodash")
+async = require("async")
+exec = require("child_process").exec
+
+finished_projects_path = "/tmp/finished-projects"
+all_projects_path = "/tmp/all-projects"
+project_too_large_path = "/tmp/large_projects"
+
+
+printProgress = ->
+	exec "wc #{finished_projects_path}", (error, results) ->
+		setTimeout printProgress, 1000 * 30
+
+checkIfFileHasBeenProccessed = (project_id, callback)->
+	exec "grep #{project_id} #{finished_projects_path}", (error, results) ->
+		hasBeenProcessed = _.include(results, project_id)
+		callback(error, hasBeenProcessed)
+
+loadProjectIds = (callback)->
+	console.log "loading project ids from #{all_projects_path}"
+	fs.readFile all_projects_path, "utf-8", (err, data)->
+		ids = data.split("\n")
+		console.log "loaded #{ids.length} project ids from #{all_projects_path}"
+		callback err, ids
+
+getAndWriteProjectids = (callback)->
+	console.log "finding all project id's - #{new Date().toString()}"
+	db.projects.find {}, {_id:1}, (err, ids)->
+		console.log "total found projects in mongo #{ids.length} - #{new Date().toString()}"
+		ids = _.pluck ids, '_id'
+		ids = _.filter ids, (id)-> id?
+		fileData = ids.join("\n")
+		fs.writeFile all_projects_path, fileData, ->
+			callback(err, ids)
+
+markProjectAsToLargeAndFinished = (project_id, callback)->
+	console.log "#{project_id} too large"
+	markProjectAsProcessed project_id, (err)->
+		fs.appendFile project_too_large_path, "#{project_id}\n", callback
+
+getProjectIds = (callback)->
+	exists = fs.existsSync all_projects_path
+	if exists
+		loadProjectIds callback
+	else
+		getAndWriteProjectids callback
+
+markProjectAsProcessed = (project_id, callback)->
+	fs.appendFile finished_projects_path, "#{project_id}\n", callback
+
+getAllDocs = (project_id, callback = (error, docs) ->) ->
+	db.projects.findOne _id:ObjectId(project_id), (error, project) ->
+		return callback(error) if error?
+		if !project?
+			console.log "no such project #{project_id}"
+			return callback()
+		size = require("../node_modules/mongojs/node_modules/mongodb/node_modules/bson/").BSONPure.BSON.calculateObjectSize(project)
+		if size > 12000000 #12mb
+			return markProjectAsToLargeAndFinished project_id, callback
+		findAllDocsInProject project, (error, docs) ->
+			return callback(error) if error?
+			return callback null, docs
+
+findAllDocsInProject = (project, callback = (error, docs) ->) ->
+	callback null, _findAllDocsInFolder project.rootFolder[0]
+
+_findDocInFolder = (folder = {}, doc_id, currentPath) ->
+	for doc, i in folder.docs or []
+		if doc?._id? and doc._id.toString() == doc_id.toString()
+			return {
+				doc: doc
+				mongoPath: "#{currentPath}.docs.#{i}"
+			}
+
+	for childFolder, i in folder.folders or []
+		result = _findDocInFolder childFolder, doc_id, "#{currentPath}.folders.#{i}"
+		return result if result?
+
+	return null
+
+_findAllDocsInFolder = (folder = {}) ->
+	docs = folder.docs or []
+	for childFolder in folder.folders or []
+		docs = docs.concat _findAllDocsInFolder childFolder
+	return docs
+
+insertDocIntoDocCollection = (project_id, doc_id, lines, oldRev, callback)->
+	if !project_id?
+		return callback("no project id")
+	if !doc_id?
+		return callback()
+	if !lines?
+		lines = [""]
+	update = {}
+	update["_id"] = ObjectId(doc_id.toString())
+	update["lines"] = lines
+	update["project_id"] = ObjectId(project_id)
+	update["rev"] = oldRev || 0
+	db.docs.insert update, callback
+
+saveDocsIntoMongo = (project_id, docs, callback)->
+	jobs = _.map docs, (doc)->
+		(cb)->
+			if !doc?
+				console.error "null doc in project #{project_id}" #just skip it, not a big deal
+				return cb()
+			insertDocIntoDocCollection project_id, doc._id, doc.lines, doc.rev, (err)->
+				if err?.code == 11000 #duplicate key, doc already in there so its not a problem.
+					err = undefined
+				if err?
+					console.log "error inserting doc into doc collection", err
+				cb(err)
+
+
+	async.series jobs, callback
+
+
+processNext = (project_id, callback)->
+	checkIfFileHasBeenProccessed project_id, (err, hasBeenProcessed)->
+		if hasBeenProcessed
+			console.log "#{project_id} already procssed, skipping"
+			return callback()
+		console.log "#{project_id} processing"
+		getAllDocs project_id, (err, docs)->
+			if err?
+				console.error err, project_id, "could not get all docs"
+				return callback(err)
+			else
+				saveDocsIntoMongo project_id, docs, (err)->
+					if err?
+						console.error err, project_id, "could not save docs into mongo"
+						return callback(err)
+					markProjectAsProcessed project_id, (err)->
+						setTimeout(
+							-> callback(err)
+						,500)
+
+
+
+exports.migrate = (client, done = ->)->
+	getProjectIds (err, ids)->
+		printProgress()
+		jobs = _.map ids, (id)->
+			return (cb)->
+				processNext(id, cb)
+		async.series jobs, (err)->
+			if err?
+				console.error err, "at end of jobs"
+			else
+				console.log "finished"
+			done(err)
+
+
+exports.rollback = (next)->
+	next()

migrations/about_migrations.md

@@ -0,0 +1,2 @@
+* if migration is stopped mid way it will start at the beginging next time
+* to see the run migrations do db.getCollection('_migrations').find() you can't do db._migrations.find()

package.json

@@ -1,11 +1,17 @@
 {
   "name": "sharelatex",
-  "version": "0.1.1",
+  "version": "0.1.4",
   "description": "An online collaborative LaTeX editor",
   "dependencies": {
     "async": "^0.9.0",
+    "east": "^0.2.3",
+    "east-mongo": "^0.1.2",
+    "grunt-shell": "^1.1.1",
+    "lodash": "^3.0.0",
+    "mongojs": "^0.18.1",
     "rimraf": "~2.2.6",
-    "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git"
+    "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git",
+    "underscore": "^1.7.0"
   },
   "devDependencies": {
     "grunt": "~0.4.2",
@@ -14,6 +20,7 @@
     "grunt-execute": "~0.1.5",
     "grunt-available-tasks": "~0.4.1",
     "grunt-concurrent": "~0.4.3",
+    "grunt-contrib-coffee": "~0.10.1",
     "semver": "~2.2.1",
     "knox": "~0.8.9"
   }

package/config/settings.coffee

@@ -1,219 +0,0 @@
-Path = require('path')
-
-# These credentials are used for authenticating api requests
-# between services that may need to go over public channels
-httpAuthUser = "sharelatex"
-httpAuthPass = "CRYPTO_RANDOM" # Randomly generated for you
-httpAuthUsers = {}
-httpAuthUsers[httpAuthUser] = httpAuthPass
-
-DATA_DIR = '/var/lib/sharelatex/data'
-TMP_DIR = '/var/lib/sharelatex/tmp'
-
-module.exports =
-	# Databases
-	# ---------
-
-	# ShareLaTeX's main persistant data store is MongoDB (http://www.mongodb.org/)
-	# Documentation about the URL connection string format can be found at:
-	#
-	#    http://docs.mongodb.org/manual/reference/connection-string/
-	# 
-	# The following works out of the box with Mongo's default settings:
-	mongo:
-		url : 'mongodb://127.0.0.1/sharelatex'
-
-	# Redis is used in ShareLaTeX for high volume queries, like real-time
-	# editing, and session management.
-	#
-	# The following config will work with Redis's default settings:
-	redis:
-		web:
-			host: "localhost"
-			port: "6379"
-			password: ""
-
-	# The compile server (the clsi) uses a SQL database to cache files and
-	# meta-data. sqllite is the default, and the load is low enough that this will
-	# be fine in production (we use sqllite at sharelatex.com).
-	#
-	# If you want to configure a different database, see the Sequelize documentation
-	# for available options:
-	#
-	#    https://github.com/sequelize/sequelize/wiki/API-Reference-Sequelize#example-usage
-	#
-	mysql:
-		clsi:
-			database: "clsi"
-			username: "clsi"
-			password: ""
-			dialect: "sqlite"
-			storage: Path.join(DATA_DIR, "db.sqlite")
-
-	# File storage
-	# ------------
-
-	# ShareLaTeX can store binary files like images either locally or in Amazon
-	# S3. The default is locally:
-	filestore:
-		backend: "fs"	
-		stores:
-			user_files: Path.join(DATA_DIR, "user_files")
-			
-	# To use Amazon S3 as a storage backend, comment out the above config, and
-	# uncomment the following, filling in your key, secret, and bucket name:
-	#
-	# filestore:
-	# 	backend: "s3"
-	# 	stores:
-	# 		user_files: "BUCKET_NAME"
-	# 	s3:
-	# 		key: "AWS_KEY"
-	# 		secret: "AWS_SECRET"
-	# 		
-
-	# Local disk caching
-	# ------------------
-	path:
-		# If we ever need to write something to disk (e.g. incoming requests
-		# that need processing but may be too big for memory), then write
-		# them to disk here:
-		dumpFolder:   Path.join(TMP_DIR, "dumpFolder")
-		# Where to write uploads before they are processed
-		uploadFolder: Path.join(TMP_DIR, "uploads")
-		# Where to write the project to disk before running LaTeX on it
-		compilesDir:  Path.join(DATA_DIR, "compiles")
-		# Where to cache downloaded URLs for the CLSI
-		clsiCacheDir: Path.join(DATA_DIR, "cache")
-
-	# Server Config
-	# -------------
-
-	# Where your instance of ShareLaTeX can be found publicly. This is used
-	# when emails are sent out and in generated links:
-	siteUrl : 'http://localhost:3000'
-	
-	# If provided, a sessionSecret is used to sign cookies so that they cannot be
-	# spoofed. This is recommended.
-	security:
-		sessionSecret: "CRYPTO_RANDOM" # This was randomly generated for you
-
-	# These credentials are used for authenticating api requests
-	# between services that may need to go over public channels
-	httpAuthUsers: httpAuthUsers
-	
-	# Should javascript assets be served minified or not. Note that you will
-	# need to run `grunt compile:minify` within the web-sharelatex directory
-	# to generate these.
-	useMinifiedJs: false
-
-	# Should static assets be sent with a header to tell the browser to cache
-	# them. This should be false in development where changes are being made,
-	# but should be set to true in production.
-	cacheStaticAssets: false
-
-	# If you are running ShareLaTeX over https, set this to true to send the
-	# cookie with a secure flag (recommended).
-	secureCookie: false
-	
-	# If you are running ShareLaTeX behind a proxy (like Apache, Nginx, etc)
-	# then set this to true to allow it to correctly detect the forwarded IP
-	# address and http/https protocol information.
-	behindProxy: false
-
-	# Sending Email
-	# -------------
-	#
-	# You must configure a mail server to be able to send invite emails from
-	# ShareLaTeX. The config settings are passed to nodemailer. See the nodemailer
-	# documentation for available options:
-	#
-	#     http://www.nodemailer.com/docs/transports
-	#
-	# email:
-	#	fromAddress: ""
-	#	replyTo: ""
-	#	transport: "SES"
-	#	parameters:
-	#		AWSAccessKeyID: ""
-	#		AWSSecretKey: ""
-
-	# Spell Check Languages
-	# ---------------------
-	#
-	# You must have the corresponding aspell dictionary installed to 
-	# be able to use a language. Run `grunt check:aspell` to check which
-	# dictionaries you have installed. These should be set for the `code` for
-	# each language.
-	languages: [
-		{name: "English", code: "en"}
-	]
-	
-	# Service locations
-	# -----------------
-
-	# ShareLaTeX is comprised of many small services, which each expose
-	# an HTTP API running on a different port. Generally you
-	# can leave these as they are unless you have some other services
-	# running which conflict, or want to run the web process on port 80.
-	# internal:
-	# 	web:
-	# 		port: webPort = 3000
-	# 		host: "localhost"
-	# 	documentupdater:
-	# 		port: docUpdaterPort = 3003
-	# 		host: "localhost"
-	# 	filestore:
-	# 		port: filestorePort = 3009
-	# 		host: "localhost"
-	# 	chat:
-	# 		port: chatPort = 3010
-	# 		host: "localhost"
-	# 	tags:
-	# 		port: tagsPort = 3012
-	# 		host: "localhost"
-	# 	clsi:
-	# 		port: clsiPort = 3013
-	# 		host: "localhost"
-	# 	trackchanges:
-	# 		port: trackchangesPort = 3015
-	# 		host: "localhost"
-	# 	docstore:
-	# 		port: docstorePort = 3016
-	# 		host: "localhost"
-	# 	spelling:
-	# 		port: spellingPort = 3005
-	# 		host: "localhost"
-
-	# If you change the above config, or run some services on remote servers,
-	# you need to tell the other services where to find them:
-	apis:
-		web:
-			url: "http://localhost:3000"
-			user: httpAuthUser
-			pass: httpAuthPass
-	# 	documentupdater:
-	# 		url : "http://localhost:#{docUpdaterPort}"
-	# 	clsi:
-	# 		url: "http://localhost:#{clsiPort}"
-	# 	filestore:
-	# 		url: "http://localhost:#{filestorePort}"
-	# 	trackchanges:
-	# 		url: "http://localhost:#{trackchangesPort}"
-	# 	docstore:
-	# 		url: "http://localhost:#{docstorePort}"
-	# 	tags:
-	# 		url: "http://localhost:#{tagsPort}"
-	# 	spelling:
-	# 		url: "http://localhost:#{spellingPort}"
-	# 	chat:
-	# 		url: "http://localhost:#{chatPort}"
-	
-
-# With lots of incoming and outgoing HTTP connections to different services,
-# sometimes long running, it is a good idea to increase the default number
-# of sockets that Node will hold open.
-http = require('http')
-http.globalAgent.maxSockets = 300
-https = require('https')
-https.globalAgent.maxSockets = 300

package/nginx/sharelatex.conf

@@ -15,6 +15,18 @@ server {
 		proxy_send_timeout 3m;
 	}
 	
+	location /socket.io {
+		proxy_pass http://localhost:3026;
+		proxy_set_header X-Forwarded-Proto $scheme;
+		proxy_set_header Host $http_x_forwarded_host;
+		proxy_http_version 1.1;
+		proxy_set_header Upgrade $http_upgrade;
+		proxy_set_header Connection "upgrade";
+		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+		proxy_read_timeout 3m;
+		proxy_send_timeout 3m;
+	}
+
 	location /stylesheets {
 		expires 1y;
 		root $static_path/;

package/scripts/after_install.sh

@@ -1,26 +0,0 @@
-#!/bin/sh
-# Create random secret keys (twice, once for http auth pass, once for cookie secret).
-sed -i "0,/CRYPTO_RANDOM/s/CRYPTO_RANDOM/$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 64 | head -n 1)/" /etc/sharelatex/settings.coffee
-sed -i "0,/CRYPTO_RANDOM/s/CRYPTO_RANDOM/$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 64 | head -n 1)/" /etc/sharelatex/settings.coffee
-
-sudo adduser --system --group --home /var/www/sharelatex --no-create-home sharelatex
-
-mkdir -p /var/log/sharelatex
-chown sharelatex:sharelatex /var/log/sharelatex
-
-mkdir -p /var/lib/sharelatex
-mkdir -p /var/lib/sharelatex/data/user_files
-mkdir -p /var/lib/sharelatex/tmp/uploads
-mkdir -p /var/lib/sharelatex/data/compiles
-mkdir -p /var/lib/sharelatex/data/cache
-mkdir -p /var/lib/sharelatex/tmp/dumpFolder
-chown -R sharelatex:sharelatex /var/lib/sharelatex
-service sharelatex-web restart
-service sharelatex-document-updater restart
-service sharelatex-clsi restart
-service sharelatex-filestore restart
-service sharelatex-track-changes restart
-service sharelatex-docstore restart
-service sharelatex-chat restart
-service sharelatex-tags restart
-service sharelatex-spelling restart

package/upstart/sharelatex-chat.conf

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-clsi.conf

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-docstore.conf

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-document-updater.conf

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-filestore.conf

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-real-time.conf

@@ -0,0 +1,28 @@
+description "sharelatex-web"
+author      "ShareLaTeX <team@sharelatex.com>"
+
+start on (local-filesystems and net-device-up IFACE!=lo)
+stop on shutdown
+
+respawn
+
+limit nofile 8192 8192
+
+pre-start script
+    mkdir -p /var/log/sharelatex
+end script
+
+script
+    SERVICE=real-time
+    USER=sharelatex
+    GROUP=sharelatex
+    # You may need to replace this with an absolute 
+    # path to Node.js if it's not in your system PATH.
+    NODE=node
+    SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee
+    LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
+
+    echo $$ > /var/run/sharelatex-$SERVICE.pid
+    cd /var/www/sharelatex/$SERVICE
+    exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
+end script

package/upstart/sharelatex-spelling.conf

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-tags.conf

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-template.conf

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-track-changes.conf

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-web.conf

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script