Release 0.1.3

force an npm rebuild to work around issue #129, missing bcrypt bindings

.gitignore

@@ -1,4 +1,4 @@
-config
+/config
 config-local
 node_modules
 
@@ -8,12 +8,16 @@ clsi
 filestore
 track-changes
 docstore
+tags
+chat
+spelling
+real-time
+migrations/*.js
 
-compiles
-cache
-user_files
-template_files
+data
+tmp
 
 db.sqlite
 
+.DS_Store
 .vagrant

CHANGELOG.md

@@ -0,0 +1,21 @@
+v0.1.3
+------
+
+* Fix bug with large files being corrupted when downloaded.
+* Update Ace editor to lastest release.
+* Lots of added null checks in the front-end javascript.
+* Don't crash if 'unzip' program isn't present.
+* Allow track-changes history to be packed into compressed 'packs'. This must be done manually for now.
+* Escape any shell special characters in the CLSI root path.
+
+v0.1.2
+------
+
+* Re-brand open-source ShareLaTeX code as 'ShareLaTeX Community Edition'.
+* The Dropbox and template code has been extracted out into a separate module and removed from the ShareLaTeX Community Edition. There should be no broken features due to lack of open source components now.
+* Websockets and real-time data now go to a separate light-weight [real-time](https://github.com/sharelatex/real-time) service.
+* Updated PDF viewer that loads page-by-page for much quicker loading times on large documents.
+* Links are clickable in chat messages.
+* Mathjax libraries are now served locally.
+* Optimisation of Angular digest loop in editor should reduce CPU usage in certain cases.
+* Numerous small bug fixes.

Gruntfile.coffee

@@ -5,25 +5,51 @@ rimraf = require "rimraf"
 Path = require "path"
 semver = require "semver"
 knox = require "knox"
+crypto = require "crypto"
+async = require "async"
+settings = require("settings-sharelatex")
+
 
 SERVICES = [{
 	name: "web"
 	repo: "https://github.com/sharelatex/web-sharelatex.git"
+	version: "v0.1.3"
+}, {
+	name: "real-time"
+	repo: "https://github.com/sharelatex/real-time-sharelatex.git"
+	version: "v0.1.3"
 }, {
 	name: "document-updater"
 	repo: "https://github.com/sharelatex/document-updater-sharelatex.git"
+	version: "v0.1.3"
 }, {
 	name: "clsi"
 	repo: "https://github.com/sharelatex/clsi-sharelatex.git"
+	version: "v0.1.3"
 }, {
 	name: "filestore"
 	repo: "https://github.com/sharelatex/filestore-sharelatex.git"
+	version: "v0.1.3"
 }, {
 	name: "track-changes"
 	repo: "https://github.com/sharelatex/track-changes-sharelatex.git"
+	version: "v0.1.3"
 }, {
 	name: "docstore"
 	repo: "https://github.com/sharelatex/docstore-sharelatex.git"
+	version: "v0.1.3"
+}, {
+	name: "chat"
+	repo: "https://github.com/sharelatex/chat-sharelatex.git"
+	version: "v0.1.3"
+}, {
+	name: "tags"
+	repo: "https://github.com/sharelatex/tags-sharelatex.git"
+	version: "v0.1.3"
+}, {
+	name: "spelling"
+	repo: "https://github.com/sharelatex/spelling-sharelatex.git"
+	version: "v0.1.3"
 }]
 
 module.exports = (grunt) ->
@@ -31,6 +57,9 @@ module.exports = (grunt) ->
 	grunt.loadNpmTasks 'grunt-execute'
 	grunt.loadNpmTasks 'grunt-available-tasks'
 	grunt.loadNpmTasks 'grunt-concurrent'
+	grunt.loadNpmTasks "grunt-contrib-coffee"
+	grunt.loadNpmTasks "grunt-shell"
+
 
 	execute = {}
 	for service in SERVICES
@@ -46,6 +75,20 @@ module.exports = (grunt) ->
 				options:
 					limit: SERVICES.length
 					logConcurrentOutput: true
+		coffee:
+			migrate: 
+				expand: true,
+				flatten: false,
+				cwd: './',
+				src: ['./migrations/*.coffee'],
+				dest: './',
+				ext: '.js'
+				options:
+					bare:true
+
+		shell:
+			migrate:
+				command: "./node_modules/east/bin/east migrate --adapter east-mongo --url #{settings?.mongo?.url}"
 
 		availabletasks:
 			tasks:
@@ -66,7 +109,7 @@ module.exports = (grunt) ->
 						"Misc": [
 							"help"
 						]
-						"Install tasks": ("install:#{service.name}" for service in SERVICES).concat(["install:all", "install", "install:config"])
+						"Install tasks": ("install:#{service.name}" for service in SERVICES).concat(["install:all", "install", "install:dirs", "install:config"])
 						"Update tasks": ("update:#{service.name}" for service in SERVICES).concat(["update:all", "update"])
 						"Config tasks": ["install:config"]
 						"Checks": ["check", "check:redis", "check:latexmk", "check:s3", "check:make"]
@@ -75,18 +118,23 @@ module.exports = (grunt) ->
 		do (service) ->
 			grunt.registerTask "install:#{service.name}", "Download and set up the #{service.name} service", () ->
 				done = @async()
-				Helpers.installService(service.repo, service.name, done)
+				Helpers.installService(service, done)
 			grunt.registerTask "update:#{service.name}", "Checkout and update the #{service.name} service", () ->
 				done = @async()
-				Helpers.updateService(service.name, done)
+				Helpers.updateService(service, done)
 			grunt.registerTask "run:#{service.name}", "Run the ShareLaTeX #{service.name} service", ["bunyan", "execute:#{service.name}"]
+			grunt.registerTask "release:#{service.name}", "Create a new release version of #{service.name} (specify with --release option)", () ->
+				done = @async()
+				Helpers.createNewRelease(service, grunt.option("release"), done)
 
 	grunt.registerTask 'install:config', "Copy the example config into the real config", () ->
 		Helpers.installConfig @async()
+	grunt.registerTask 'install:dirs', "Copy the example config into the real config", () ->
+		Helpers.createDataDirs @async()
 	grunt.registerTask 'install:all', "Download and set up all ShareLaTeX services",
 		["check:make"].concat(
 			("install:#{service.name}" for service in SERVICES)
-		).concat(["install:config"])
+		).concat(["install:config", "install:dirs"])
 	grunt.registerTask 'install', 'install:all'
 	grunt.registerTask 'update:all', "Checkout and update all ShareLaTeX services",
 		["check:make"].concat(
@@ -107,63 +155,133 @@ module.exports = (grunt) ->
 		Helpers.checkS3 @async()
 	grunt.registerTask "check:fs", "Check that local filesystem options are configured", () ->
 		Helpers.checkFS @async()
+	grunt.registerTask "check:aspell", "Check that aspell is installed", () ->
+		Helpers.checkAspell @async()
 	grunt.registerTask "check:make", "Check that make is installed", () ->
 		Helpers.checkMake @async()
-	grunt.registerTask "check", "Check that you have the required dependencies installed", ["check:redis", "check:latexmk", "check:s3", "check:fs"]
+	grunt.registerTask "check", "Check that you have the required dependencies installed", ["check:redis", "check:latexmk", "check:s3", "check:fs", "check:aspell"]
 
-	grunt.registerTask "build_deb", "Build an installable .deb file from the current directory", () ->
+	grunt.registerTask "build:deb", "Build an installable .deb file from the current directory", () ->
 		Helpers.buildDeb @async()
+	grunt.registerTask "build:upstart_scripts", "Create upstart scripts for each service", () ->
+		Helpers.buildUpstartScripts()
+
+
+	#grunt.registerTask 'migrate', "compile migrations and run them", ['coffee:migrate', 'shell:migrate']
+
 
 	Helpers =
-		installService: (repo_src, dir, callback = (error) ->) ->
-			Helpers.cloneGitRepo repo_src, dir, (error) ->
+		installService: (service, callback = (error) ->) ->
+			Helpers.cloneGitRepo service, (error) ->
 				return callback(error) if error?
-				Helpers.installNpmModules dir, (error) ->
+				Helpers.installNpmModules service, (error) ->
 					return callback(error) if error?
-					Helpers.runGruntInstall dir, (error) ->
+					Helpers.rebuildNpmModules service, (error) ->
 						return callback(error) if error?
-						callback()
+						Helpers.runGruntInstall service, (error) ->
+							return callback(error) if error?
+							callback()
 
-		updateService: (dir, callback = (error) ->) ->
-			Helpers.updateGitRepo dir, (error) ->
+		updateService: (service, callback = (error) ->) ->
+			Helpers.updateGitRepo service, (error) ->
 				return callback(error) if error?
-				Helpers.installNpmModules dir, (error) ->
+				Helpers.installNpmModules service, (error) ->
 					return callback(error) if error?
-					Helpers.runGruntInstall dir, (error) ->
+					Helpers.rebuildNpmModules service, (error) ->
 						return callback(error) if error?
-						callback()
+						Helpers.runGruntInstall service, (error) ->
+							return callback(error) if error?
+							callback()
 
-		cloneGitRepo: (repo_src, dir, callback = (error) ->) ->
+		cloneGitRepo: (service, callback = (error) ->) ->
+			repo_src = service.repo
+			dir = service.name
 			if !fs.existsSync(dir)
-				proc = spawn "git", ["clone", repo_src, dir], stdio: "inherit"
+				proc = spawn "git", [
+					"clone",
+					"-b", service.version,
+					repo_src,
+					dir
+				], stdio: "inherit"
 				proc.on "close", () ->
 					callback()
 			else
 				console.log "#{dir} already installed, skipping."
 				callback()
 
-		updateGitRepo: (dir, callback = (error) ->) ->
-			proc = spawn "git", ["checkout", "master"], cwd: dir, stdio: "inherit"
+		updateGitRepo: (service, callback = (error) ->) ->
+			dir = service.name
+			proc = spawn "git", ["checkout", service.version], cwd: dir, stdio: "inherit"
 			proc.on "close", () ->
 				proc = spawn "git", ["pull"], cwd: dir, stdio: "inherit"
 				proc.on "close", () ->
 					callback()
+					
+		createNewRelease: (service, version, callback = (error) ->) ->
+			dir = service.name
+			proc = spawn "sed", [
+				"-i", "",
+				"s/\"version\".*$/\"version\": \"#{version}\",/g",
+				"package.json"
+			], cwd: dir, stdio: "inherit"
+			proc.on "close", () ->
+				proc = spawn "git", ["commit", "-a", "-m", "Release version #{version}"], cwd: dir, stdio: "inherit"
+				proc.on "close", () ->
+					proc = spawn "git", ["tag", "v#{version}"], cwd: dir, stdio: "inherit"
+					proc.on "close", () ->
+						proc = spawn "git", ["push"], cwd: dir, stdio: "inherit"
+						proc.on "close", () ->
+							proc = spawn "git", ["push", "--tags"], cwd: dir, stdio: "inherit"
+							proc.on "close", () ->
+								callback()
 
-		installNpmModules: (dir, callback = (error) ->) ->
+		installNpmModules: (service, callback = (error) ->) ->
+			dir = service.name
 			proc = spawn "npm", ["install"], stdio: "inherit", cwd: dir
 			proc.on "close", () ->
 				callback()
 
+		# work around for https://github.com/npm/npm/issues/5400
+		# where binary modules are not built due to bug in npm
+		rebuildNpmModules: (service, callback = (error) ->) ->
+			dir = service.name
+			proc = spawn "npm", ["rebuild"], stdio: "inherit", cwd: dir
+			proc.on "close", () ->
+				callback()
+
+		createDataDirs: (callback = (error) ->) ->
+			DIRS = [
+				"tmp/dumpFolder"
+				"tmp/uploads"
+				"data/user_files"
+				"data/compiles"
+				"data/cache"
+			]
+			jobs = []
+			for dir in DIRS
+				do (dir) ->
+					jobs.push (callback) ->
+						path = Path.join(__dirname, dir)
+						grunt.log.writeln "Ensuring '#{path}' exists"
+						exec "mkdir -p #{path}", callback
+			async.series jobs, callback
+
 		installConfig: (callback = (error) ->) ->
-			if !fs.existsSync("config/settings.development.coffee")
-				grunt.log.writeln "Copying example config into config/settings.development.coffee"
-				exec "cp config/settings.development.coffee.example config/settings.development.coffee", (error, stdout, stderr) ->
-					callback(error)
+			src = "config/settings.development.coffee.example"
+			dest = "config/settings.development.coffee"
+			if !fs.existsSync(dest)
+				grunt.log.writeln "Creating config at #{dest}"
+				config = fs.readFileSync(src).toString()
+				config = config.replace /CRYPTO_RANDOM/g, () ->
+					crypto.randomBytes(64).toString("hex")
+				fs.writeFileSync dest, config
+				callback()
 			else
 				grunt.log.writeln "Config file already exists. Skipping."
 				callback()
 
-		runGruntInstall: (dir, callback = (error) ->) ->
+		runGruntInstall: (service, callback = (error) ->) ->
+			dir = service.name
 			proc = spawn "grunt", ["install"], stdio: "inherit", cwd: dir
 			proc.on "close", () ->
 				callback()
@@ -196,13 +314,18 @@ module.exports = (grunt) ->
 		checkLatexmk: (callback = (error) ->) ->
 			grunt.log.write "Checking latexmk is installed... "
 			exec "latexmk --version", (error, stdout, stderr) ->
-				if error? and error.message.match("command not found")
+				if error? and error.message.match("not found")
 					grunt.log.error "FAIL."
 					grunt.log.errorlns """
 					Either latexmk is not installed or is not in your PATH.
 
 					latexmk comes with TexLive 2013, and must be a version from 2013 or later.
-					This is a not a fatal error, but compiling will not work without latexmk
+					If you have already have TeXLive installed, then make sure it is
+					included in your PATH (example for 64-bit linux):
+					
+						export PATH=$PATH:/usr/local/texlive/2014/bin/x86_64-linux/
+					
+					This is a not a fatal error, but compiling will not work without latexmk.
 					"""
 					return callback(error)
 				else if error?
@@ -226,6 +349,34 @@ module.exports = (grunt) ->
 							"""
 							error = new Error("latexmk is too old")
 				callback(error)
+				
+		checkAspell: (callback = (error) ->) ->
+			grunt.log.write "Checking aspell is installed... "
+			exec "aspell dump dicts", (error, stdout, stderr) ->
+				if error? and error.message.match("not found")
+					grunt.log.error "FAIL."
+					grunt.log.errorlns """
+					Either aspell is not installed or is not in your PATH.
+					
+					On Ubuntu you can install aspell with:
+					
+						sudo apt-get install aspell
+						
+					Or on a mac:
+					
+						brew install aspell
+						
+					This is not a fatal error, but the spell-checker will not work without aspell
+					"""
+					return callback(error)
+				else if error?
+					return callback(error)
+				else
+					grunt.log.writeln "OK."
+					grunt.log.writeln "The following spell check dictionaries are available:"
+					grunt.log.write stdout
+					callback()
+				callback(error)
 
 		checkS3: (callback = (error) ->) ->
 			Settings = require "settings-sharelatex"
@@ -260,7 +411,7 @@ module.exports = (grunt) ->
 						Could not connect to Amazon S3. Please check your credentials.
 						"""
 					else
-						grunt.log.write "OK."
+						grunt.log.writeln "OK."
 					callback()
 			else
 				grunt.log.writeln "Filestore other than S3 configured. Not checking S3."
@@ -269,34 +420,34 @@ module.exports = (grunt) ->
 		checkFS: (callback = (error) ->) ->
 			Settings = require "settings-sharelatex"
 			if Settings.filestore.backend=="fs"
-				grunt.log.write "Checking FS configuration..."
+				grunt.log.write "Checking FS configuration... "
 				fs = require("fs")
 				fs.exists Settings.filestore.stores.user_files, (exists) ->
 					if exists
-						grunt.log.write "OK."
+						grunt.log.writeln "OK."
 					else
 						grunt.log.error "FAIL."
 						grunt.log.errorlns """
 						Could not find directory "#{Settings.filestore.stores.user_files}". 
 						Please check your configuration.
 						"""
+					callback()
 			else
 				grunt.log.writeln "Filestore other than FS configured. Not checking FS."
-			callback()
-
+				callback()
 
 		checkMake: (callback = (error) ->) ->
 			grunt.log.write "Checking make is installed... "
 			exec "make --version", (error, stdout, stderr) ->
-				if error? and error.message.match("command not found")
+				if error? and error.message.match("not found")
 					grunt.log.error "FAIL."
 					grunt.log.errorlns """
 					Either make is not installed or is not in your path.
-
+					
 					On Ubuntu you can install make with:
-
+					
 					    sudo apt-get install build-essential
-
+					
 					"""
 					return callback(error)
 				else if error?
@@ -305,51 +456,74 @@ module.exports = (grunt) ->
 					grunt.log.write "OK."
 					return callback()
 
+		buildUpstartScripts: () ->
+			template = fs.readFileSync("package/upstart/sharelatex-template").toString()
+			for service in SERVICES
+				fs.writeFileSync "package/upstart/sharelatex-#{service.name}", template.replace(/__SERVICE__/g, service.name)
+
+		buildPackageSettingsFile: () ->
+			config = fs.readFileSync("config/settings.development.coffee.example").toString()
+			config = config.replace /DATA_DIR.*/, "DATA_DIR = '/var/lib/sharelatex/data'"
+			config = config.replace /TMP_DIR.*/, "TMP_DIR = '/var/lib/sharelatex/tmp'"
+			fs.writeFileSync "package/config/settings.coffee", config
+
 		buildDeb: (callback = (error) ->) ->
-			# TODO: filestore uses local 'uploads' directory, not configurable in settings
-			command = ["fpm", "-s", "dir", "-t", "deb", "-n", "sharelatex", "-v", "0.0.1", "--verbose"]
+			command = ["-s", "dir", "-t", "deb", "-n", "sharelatex", "-v", "0.0.1", "--verbose"]
 			command.push(
-				"--maintainer", "'ShareLaTeX <team@sharelatex.com>'"
-				"--config-files", "/etc/sharelatex/settings.coffee",
-				"--directories",  "/var/data/sharelatex"
+				"--maintainer", "ShareLaTeX <team@sharelatex.com>"
+				"--config-files", "/etc/sharelatex/settings.coffee"
+				"--config-files", "/etc/nginx/conf.d/sharelatex.conf"
+				"--directories",  "/var/lib/sharelatex"
 				"--directories",  "/var/log/sharelatex"
 			)
 
 			command.push(
-				"--depends", "'redis-server > 2.6.12'"
-				"--depends", "'mongodb-10gen > 2.4.0'"
-				"--depends", "'nodejs > 0.10.0'"
+				"--depends", "redis-server > 2.6.12"
+				"--depends", "mongodb-org > 2.4.0"
+				"--depends", "nodejs > 0.10.0"
 			)
+			
+			@buildPackageSettingsFile()
 
-			template = fs.readFileSync("package/upstart/sharelatex-template").toString()
+			@buildUpstartScripts()
 			for service in SERVICES
-				fs.writeFileSync "package/upstart/sharelatex-#{service.name}", template.replace(/SERVICE/g, service.name)
 				command.push(
 					"--deb-upstart", "package/upstart/sharelatex-#{service.name}"
 				)
 
 			after_install_script = """
 				#!/bin/sh
+				# Create random secret keys (twice, once for http auth pass, once for cookie secret).
+				sed -i "0,/CRYPTO_RANDOM/s/CRYPTO_RANDOM/$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 64 | head -n 1)/" /etc/sharelatex/settings.coffee
+				sed -i "0,/CRYPTO_RANDOM/s/CRYPTO_RANDOM/$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 64 | head -n 1)/" /etc/sharelatex/settings.coffee
+				
 				sudo adduser --system --group --home /var/www/sharelatex --no-create-home sharelatex
 
 				mkdir -p /var/log/sharelatex
 				chown sharelatex:sharelatex /var/log/sharelatex
+				
+				mkdir -p /var/lib/sharelatex
 
 			"""
 
-			for dir in ["user_files", "uploads", "compiles", "cache", "dump"]
+			for dir in ["data/user_files", "tmp/uploads", "data/compiles", "data/cache", "tmp/dumpFolder"]
 				after_install_script += """
-					mkdir -p /var/data/sharelatex/#{dir}
-					chown sharelatex:sharelatex /var/data/sharelatex/#{dir}
+					mkdir -p /var/lib/sharelatex/#{dir}
 					
 				"""
+			
+			after_install_script += """
+				chown -R sharelatex:sharelatex /var/lib/sharelatex
+				
+			"""	
 
 			for service in SERVICES
 				after_install_script += "service sharelatex-#{service.name} restart\n"
 			fs.writeFileSync "package/scripts/after_install.sh", after_install_script
 			command.push("--after-install", "package/scripts/after_install.sh")
 
-			command.push("--exclude", "'**/.git'")
+			command.push("--exclude", "**/.git")
+			command.push("--exclude", "**/node_modules/grunt-*")
 			for path in ["filestore/user_files", "filestore/uploads", "clsi/cache", "clsi/compiles"]
 				command.push "--exclude", path
 
@@ -358,13 +532,15 @@ module.exports = (grunt) ->
 
 			command.push(
 				"package/config/settings.coffee=/etc/sharelatex/settings.coffee"
+				"package/nginx/sharelatex=/etc/nginx/conf.d/sharelatex.conf"
 			)
-			console.log command.join(" ")
-			exec command.join(" "), (error, stdout, stderr) ->
-				return callback(error) if error?
-				console.log stdout
-				console.error stderr if stderr?
-				callback()
+			console.log "fpm " + command.join(" ")
+			proc = spawn "fpm", command, stdio: "inherit"
+			proc.on "close", (code) ->
+				if code != 0
+					callback(new Error("exit code: #{code}"))
+				else
+					callback()
 
 			
 

README.md

@@ -1,58 +1,25 @@
 ShareLaTeX
 ==========
 
-[ShareLaTeX](https://www.sharelatex.com) is now open source! ShareLaTeX is an online real-time collaborative LaTeX editor, and you can now run your own local version where you can host, edit, collaborate in real-time, and compile your LaTeX documents. We’re still 100% focused on running the hosted version at http://www.sharelatex.com, but we want to be more flexible in how you can use ShareLaTeX, and give something back to our wonderful community.
+[ShareLaTeX](https://www.sharelatex.com) is an open-source online real-time collaborative LaTeX editor. We run a hosted version at http://www.sharelatex.com, but you can also run your own local version, and contribute to the development of ShareLaTeX.
 
-**[Read more on our blog](https://www.sharelatex.com/blog/2014/02/21/sharelatex-is-now-open-source.html#.UwcnsEJ_ugc)**
+*[If you want help installing and maintaining ShareLaTeX at your university or workplace, we offer an officially supported version called ShareLaTeX Server Pro. It also comes with extra security and admin features. Click here to find out more!](https://www.sharelatex.com/university/onsite.html)*
 
 Installation
 ------------
 
-**[Please help us make ShareLaTeX as easy to install as possible by answering our quick survey about your system and needs](https://sharelatex.typeform.com/to/PLNits)**
+We have detailed installation instructions in our wiki:
 
-We're still figuring out the easiest way to let you install ShareLaTeX and get up and running quickly. If you fill in the above survey in we will be eternally grateful and it will help us make this install process as smooth as possible. For now, here is the best ways:
+* [Installing ShareLaTeX in Production](https://github.com/sharelatex/sharelatex/wiki/Production-Installation-Instructions)
+* [Setting up a ShareLaTeX Development Environment](https://github.com/sharelatex/sharelatex/wiki/Setting-up-a-Development-Environment)
 
-### Manually
+**If you have any problems, have a look at our page of [Frequent Problems and Questions](https://github.com/sharelatex/sharelatex/wiki/FAQ).**
 
-First, check out a local copy of this repository:
+Upgrading
+---------
 
-```bash
-git clone https://github.com/sharelatex/sharelatex.git
-cd sharelatex
-```
+If you are upgrading from a previous version of ShareLaTeX, please see the [Release Notes section on the Wiki] (https://github.com/sharelatex/sharelatex/wiki/Home) for all of the versions between your current version and the version you are upgrading to.
 
-Next install all the node modules and ShareLaTeX services:
-
-```bash
-npm install
-grunt install
-```
-
-This will create a config file in `config/settings.development.coffee`. You should open
-this now and configure your AWS S3 credentials, and other custom settings.
-
-Now check that your system is set up correctly to run ShareLaTeX (checks that you have
-the required dependencies installed.) Watch out for any failures.
-
-```bash
-grunt check --force
-```
-
-When that has finished, run ShareLaTeX with
-
-```bash
-grunt run
-```
-
-ShareLaTeX should now be running at http://localhost:3000.
-
-### With Vagrant
-
-There is a Vagrant and Ansible backed VM installation script for ShareLaTeX, maintained by [@palkan](https://github/palkan), available here: https://github.com/palkan/sharelatex-vagrant-ansible
-
-### With Docker
-
-An [automatic docker-based installer](https://github.com/tiagoboldt/sharelatex-docker) is available. It depends on docker and will build a production environment for running ShareLaTeX on any supported platform.
 
 Dependencies
 ------------
@@ -64,18 +31,12 @@ ShareLaTeX should run on OS X and Linux. You need:
 * A local instance of [Redis](http://redis.io/topics/quickstart) (version 2.6.12 or later) and [MongoDB](http://docs.mongodb.org/manual/installation/) running on their standard ports.
 * [TeXLive](https://www.tug.org/texlive/) 2013 or later with the `latexmk` program installed.
 
-Config
-------
-
-ShareLaTeX should run out of the box, but if you want to adjust any settings you can do so by
-editing the `config/settings.development.coffee` file. Available options are explained inline.
+ShareLaTeX needs a minimum of 2gb of memory, it is likely to be more than that though depending on usage.
 
 Other repositories
 ------------------
 
-ShareLaTeX consists of many separate services, each with their own Node.js process
-and source code repository. These are all downloaded and set upwhen you run
-`grunt install`
+This repository does not contain any code. It acts a wrapper and toolkit for managing the many different ShareLaTeX  services. These each run as their own Node.js process and have their own Github repository. These are all downloaded and set up when you run `grunt install`
 
 The different services are:
 
@@ -97,6 +58,11 @@ modifications.
 The Common LaTeX Service Interface (CLSI) which provides an API for compiling LaTeX 
 documents.
 
+### [docstore](https://github.com/sharelatex/docstore-sharelatex) [![Build Status](https://travis-ci.org/sharelatex/docstore-sharelatex.png?branch=master)](https://travis-ci.org/sharelatex/docstore-sharelatex)
+
+An API for performing CRUD (Create, Read, Update and Delete) operations on text files
+stored in ShareLaTeX.
+
 ### [filestore](https://github.com/sharelatex/filestore-sharelatex) [![Build Status](https://travis-ci.org/sharelatex/filestore-sharelatex.png?branch=master)](https://travis-ci.org/sharelatex/filestore-sharelatex)
 
 An API for performing CRUD (Create, Read, Update and Delete) operations on binary files
@@ -105,7 +71,24 @@ An API for performing CRUD (Create, Read, Update and Delete) operations on binar
 ### [track-changes](https://github.com/sharelatex/track-changes-sharelatex) [![Build Status](https://travis-ci.org/sharelatex/track-changes-sharelatex.png?branch=master)](https://travis-ci.org/sharelatex/track-changes-sharelatex)
 
 An API for compressing and storing the updates applied to a document, and then rendering a diff of the changes
-between any two time points. *Still in development and not hooked into the UI yet*.
+between any two time points.
+
+### [chat](https://github.com/sharelatex/chat-sharelatex) [![Build Status](https://travis-ci.org/sharelatex/chat-sharelatex.png?branch=master)](https://travis-ci.org/sharelatex/chat-sharelatex)
+
+The backend API for storing and fetching chat messages.
+
+### [tags](https://github.com/sharelatex/tags-sharelatex) [![Build Status](https://travis-ci.org/sharelatex/tags-sharelatex.png?branch=master)](https://travis-ci.org/sharelatex/tags-sharelatex)
+
+The backend API for managing project tags (folders).
+
+### [spelling](https://github.com/sharelatex/spelling-sharelatex)
+
+An API for running server-side spelling checking on ShareLaTeX documents.
+
+Dropbox
+-------
+
+Please note that certain features like Dropbox integration are not functional in the open source code base yet, despite appearing in the user interface. We're working on this, sorry!
 
 Contributing
 ------------

Vagrantfile

@@ -9,6 +9,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.box_url = "http://files.vagrantup.com/precise64.box"
 
   config.vm.network :forwarded_port, guest: 3000, host: 3000
+  config.vm.network :forwarded_port, guest: 80, host: 8080
 
   config.ssh.forward_agent = true
 
@@ -23,7 +24,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     chef.add_recipe 'mongodb'
     chef.add_recipe 'nodejs'
     chef.add_recipe 'texlive'
-    chef.add_recipe 'sharelatex'
+    chef.add_recipe 'packages'
 
     # You may also specify custom JSON attributes:
     chef.json = {}

chef/cookbooks/mongodb/recipes/default.rb

@@ -6,7 +6,7 @@
 #
 
 # See http://docs.mongodb.org/manual/tutorial/install-mongodb-on-ubuntu/
-apt_repository 'mongodb-10gen' do
+apt_repository 'mongodb-org' do
   uri          'http://downloads-distro.mongodb.org/repo/ubuntu-upstart'
   distribution 'dist'
   components   ['10gen']
@@ -14,6 +14,6 @@ apt_repository 'mongodb-10gen' do
   key          '7F0CEB10'
 end
 
-package 'mongodb-10gen' do
+package 'mongodb-org' do
   action :install
 end

chef/cookbooks/packages/README.md

@@ -1,5 +1,5 @@
-sharelatex Cookbook
-===================
+redis Cookbook
+==============
 TODO: Enter the cookbook description here.
 
 e.g.
@@ -11,14 +11,14 @@ TODO: List your cookbook requirements. Be sure to include any requirements this
 
 e.g.
 #### packages
-- `toaster` - sharelatex needs toaster to brown your bagel.
+- `toaster` - redis needs toaster to brown your bagel.
 
 Attributes
 ----------
 TODO: List you cookbook attributes here.
 
 e.g.
-#### sharelatex::default
+#### redis::default
 <table>
   <tr>
     <th>Key</th>
@@ -27,7 +27,7 @@ e.g.
     <th>Default</th>
   </tr>
   <tr>
-    <td><tt>['sharelatex']['bacon']</tt></td>
+    <td><tt>['redis']['bacon']</tt></td>
     <td>Boolean</td>
     <td>whether to include bacon</td>
     <td><tt>true</tt></td>
@@ -36,17 +36,17 @@ e.g.
 
 Usage
 -----
-#### sharelatex::default
+#### redis::default
 TODO: Write usage instructions for each cookbook.
 
 e.g.
-Just include `sharelatex` in your node's `run_list`:
+Just include `redis` in your node's `run_list`:
 
 ```json
 {
   "name":"my_node",
   "run_list": [
-    "recipe[sharelatex]"
+    "recipe[redis]"
   ]
 }
 ```

chef/cookbooks/packages/metadata.rb

@@ -0,0 +1,7 @@
+name             'packages'
+maintainer       'ShareLaTeX'
+maintainer_email 'team@sharelatex.com'
+license          'AGPLv3'
+description      'Installs/Configures packages'
+long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
+version          '0.1.0'

chef/cookbooks/packages/recipes/default.rb

@@ -0,0 +1,10 @@
+#
+# Cookbook Name:: packages
+# Recipe:: default
+#
+# Copyright 2014, ShareLaTeX
+#
+
+package 'git'
+package 'vim'
+package 'build-essential'

chef/cookbooks/sharelatex/CHANGELOG.md

@@ -1,12 +0,0 @@
-# CHANGELOG for sharelatex
-
-This file is used to list changes made in each version of sharelatex.
-
-## 0.1.0:
-
-* Initial release of sharelatex
-
-- - -
-Check the [Markdown Syntax Guide](http://daringfireball.net/projects/markdown/syntax) for help with Markdown.
-
-The [Github Flavored Markdown page](http://github.github.com/github-flavored-markdown/) describes the differences between markdown on github and standard markdown.

chef/cookbooks/sharelatex/metadata.rb

@@ -1,8 +0,0 @@
-name             'sharelatex'
-maintainer       'YOUR_COMPANY_NAME'
-maintainer_email 'YOUR_EMAIL'
-license          'All rights reserved'
-description      'Installs/Configures sharelatex'
-long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
-version          '0.1.0'
-depends          'texlive'

chef/cookbooks/sharelatex/providers/app.rb

@@ -1,121 +0,0 @@
-action :start do
-	package "git"
-	package "build-essential"
-
-	r = new_resource
-
-	deploy_to = "/var/www/" + r.name
-
-	node_environment = "production"
-
-	directory deploy_to do
-		user r.user if r.user
-		recursive true
-	end
-
-	env = {
-		"HOME" => deploy_to
-	}
-
-	directory "#{deploy_to}/releases" do
-		user r.user if r.user
-		recursive true
-	end
-
-	shared_dir = "#{deploy_to}/shared"
-	directory shared_dir do
-		user r.user if r.user
-		recursive true
-	end
-	directory "#{shared_dir}/config" do
-		user r.user if r.user
-		recursive true
-	end
-	directory "#{shared_dir}/log" do
-		user r.user if r.user
-		recursive true
-	end
-
-	deploy_revision deploy_to do
-		repository r.repository
-		revision r.revision
-		user r.user if r.user
-
-		purge_before_symlink [
-			"log", "config", "node_modules"
-		]
-		create_dirs_before_symlink []
-		symlinks({
-			"log"    => "log",
-			"config" => "config"
-		})
-		symlink_before_migrate({
-			"node_modules" => "node_modules"
-		})
-
-		environment env
-
-		migrate true
-		migration_command "npm install; grunt install"
-
-		before_migrate do
-			directory "#{deploy_to}/shared/node_modules" do
-				user r.user if r.user
-				recursive true
-			end
-		end
-
-		notifies :restart, "service[#{r.name}]"
-	end
-
-	env = ""
-	r.environment.each do |key, value|
-		env += "#{key}=#{value} "
-	end
-
-	file "/etc/init/#{r.name}.conf" do
-		content <<-EOS
-			description "#{r.name}"
-			author      "ShareLaTeX <team@sharelatex.com>"
-
-			start on started mountall
-			stop on shutdown
-
-			respawn
-
-			limit nofile 8192 8192
-			
-			script
-				echo $$ > /var/run/#{r.name}.pid
-				chdir #{deploy_to}/current
-				exec sudo -u #{r.user} env NODE_ENV=#{node_environment} SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee #{env} node app.js >> log/production.log 2>&1
-			end script
-		EOS
-
-		notifies :restart, "service[#{r.name}]"
-	end
-
-	directory "/etc/sharelatex"
-	template "/etc/sharelatex/settings.coffee" do
-		mode 0400
-		user "www-data"
-		notifies :restart, "service[#{r.name}]"
-	end
-
-	service "#{r.name}" do
-		provider Chef::Provider::Service::Upstart
-		action :start
-	end
-
-	file "/etc/logrotate.d/#{r.name}" do
-		content <<-EOS
-			#{deploy_to}/shared/log/*.log {
-				rotate 7
-				size 5M
-				missingok
-				compress
-				copytruncate
-			}
-		EOS
-	end
-end

chef/cookbooks/sharelatex/recipes/default.rb

@@ -1,47 +0,0 @@
-#
-# Cookbook Name:: sharelatex
-# Recipe:: default
-#
-# Copyright 2014, ShareLaTeX
-#
-
-# For filestore conversions
-package "imagemagick"
-package "optipng"
-
-for dir in ["", "compiles", "clsi-cache", "user_files"] do
-	directory "/var/lib/sharelatex/#{dir}" do
-		user  "www-data"
-		group "www-data"
-		recursive true
-	end
-end
-
-sharelatex_app "web-sharelatex" do
-	repository "https://github.com/sharelatex/web-sharelatex.git"
-	revision   "master"
-end
-
-sharelatex_app "document-updater-sharelatex" do
-	repository "https://github.com/sharelatex/document-updater-sharelatex.git"
-	revision   "master"
-end
-
-sharelatex_app "filestore-sharelatex" do
-	repository "https://github.com/sharelatex/filestore-sharelatex.git"
-	revision   "master"
-end
-
-sharelatex_app "track-changes-sharelatex" do
-	repository "https://github.com/sharelatex/track-changes-sharelatex.git"
-	revision   "master"
-end
-
-sharelatex_app "clsi-sharelatex" do
-	repository "https://github.com/sharelatex/clsi-sharelatex.git"
-	revision   "master"
-	environment({
-		"PATH" => "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:#{node[:texlive][:bin_dir]}"
-	})
-end
-

chef/cookbooks/sharelatex/resources/app.rb

@@ -1,13 +0,0 @@
-actions :start
-
-attribute :revision, :kind_of => String, :default => "master"
-attribute :repository, :kind_of => String
-attribute :user, :kind_of => String, :default => "www-data"
-attribute :group, :kind_of => String, :default => "www-data"
-attribute :environment, :kind_of => Hash, :default => {}
-
-def initialize(*args)
-	super
-	@action = :start
-end
-

chef/cookbooks/sharelatex/templates/default/settings.coffee.erb

@@ -1,268 +0,0 @@
-Path = require('path')
-http = require('http')
-http.globalAgent.maxSockets = 300
-
-# Make time interval config easier.
-seconds = 1000
-minutes = 60 * seconds
-
-# These credentials are used for authenticating api requests
-# between services that may need to go over public channels
-httpAuthUser = "sharelatex"
-httpAuthPass = "password"
-httpAuthUsers = {}
-httpAuthUsers[httpAuthUser] = httpAuthPass
-
-sessionSecret = "secret-please-change"
-
-module.exports =
-	# File storage
-	# ------------
-	#
-	# ShareLaTeX needs somewhere to store binary files like images.
-	# There are currently two options:
-	#     Your local filesystem (the default)
-	#     Amazon S3
-	filestore:
-		# which backend persistor to use.
-		# choices are 
-		# s3 - Amazon S3
-		# fs - local filesystem
-		backend: "fs"	
-		stores:
-			# where to store user and template binary files
-			#
-			# For Amazon S3 this is the bucket name to store binary files
-			#
-			# For local filesystem this is the directory to store the files in.
-			# This path must exist, not be tmpfs and be writable to by the user sharelatex is run as.
-			user_files: "/var/lib/sharelatex/user_files"
-		# Uncomment if you need to configure your S3 credentials
-		# s3:
-		# 	# if you are using S3, then fill in your S3 details below
-		# 	key: ""
-		# 	secret: ""
-
-	# Databases
-	# ---------
-	mongo:
-		url : 'mongodb://127.0.0.1/sharelatex'
-
-	redis:
-		web:
-			host: "localhost"
-			port: "6379"
-			password: ""
-
-		api:
-			host: "localhost"
-			port: "6379"
-			password: ""
-
-	mysql:
-		clsi:
-			database: "clsi"
-			username: "clsi"
-			password: ""
-			dialect: "sqlite"
-			storage: "/var/lib/sharelatex/clsi.sqlite"
-
-	# Service locations
-	# -----------------
-
-	# Configure which ports to run each service on. Generally you
-	# can leave these as they are unless you have some other services
-	# running which conflict, or want to run the web process on port 80.
-	internal:
-		web:
-			port: webPort = 3000
-			host: "localhost"
-		documentupdater:
-			port: docUpdaterPort = 3003
-			host: "localhost"
-		clsi:
-			port: clsiPort = 3013
-			host: "localhost"
-		filestore:
-			port: filestorePort = 3009
-			host: "localhost"
-		trackchanges:
-			port: trackchangesPort = 3015
-			host: "localhost"
-
-	# Tell each service where to find the other services. If everything
-	# is running locally then this is easy, but they exist as separate config
-	# options incase you want to run some services on remote hosts.
-	apis:
-		web:
-			url: "http://localhost:#{webPort}"
-			user: httpAuthUser
-			pass: httpAuthPass
-		documentupdater:
-			url : "http://localhost:#{docUpdaterPort}"
-		clsi:
-			url: "http://localhost:#{clsiPort}"
-		filestore:
-			url: "http://localhost:#{filestorePort}"
-		trackchanges:
-			url: "http://localhost:#{trackchangesPort}"
-		thirdPartyDataStore:
-			url : "http://localhost:3002"
-			emptyProjectFlushDelayMiliseconds: 5 * seconds
-		tags:
-			url :"http://localhost:3012"
-		spelling:
-			url : "http://localhost:3005"
-		versioning:
-			snapshotwaitms:3000
-			url: "http://localhost:4000"
-			username: httpAuthUser
-			password: httpAuthPass
-		recurly:
-			privateKey: ""
-			apiKey: ""
-			subdomain: ""
-		chat:
-			url: "http://localhost:3010"
-		templates:
-			port: 3007
-		blog:
-			port: 3008
-		templates_api:
-			url: "http://localhost:3007"
-
-	# Where your instance of ShareLaTeX can be found publically. Used in emails
-	# that are sent out, generated links, etc.
-	siteUrl : 'http://localhost:3000'
-
-	# Same, but with http auth credentials.
-	httpAuthSiteUrl: 'http://#{httpAuthUser}:#{httpAuthPass}@localhost:3000'
-
-	# Security
-	# --------
-	security:
-		sessionSecret: sessionSecret
-
-	httpAuthUsers: httpAuthUsers
-
-	# Default features
-	# ----------------
-	#
-	# You can select the features that are enabled by default for new
-	# new users.
-	defaultFeatures: defaultFeatures =
-		collaborators: -1
-		dropbox: true
-		versioning: true
-
-	plans: plans = [{
-		planCode: "personal"
-		name: "Personal"
-		price: 0
-		features: defaultFeatures
-	}]
-
-	# Spelling languages
-	# ------------------
-	#
-	# You must have the corresponding aspell package installed to 
-	# be able to use a language.
-	languages: [
-		{name: "English", code: "en"}
-	]
-
-	# Email support
-	# -------------
-	#
-	#	ShareLaTeX uses nodemailer (http://www.nodemailer.com/) to send transactional emails.
-	#	To see the range of transport and options they support, see http://www.nodemailer.com/docs/transports
-	#email:
-	# Who should emails be from by default?
-	#	fromAddress: ""
-	# The default replyTo field, if it should be set
-	#	replyTo: ""
-	#	lifecycle: false
-	## Example transport and parameter settings for Amazon SES
-	#	transport: "SES"
-	#	parameters:
-	#		AWSAccessKeyID: ""
-	#		AWSSecretKey: ""
-
-
-	# Third party services
-	# --------------------
-	#
-	# ShareLaTeX's regular newsletter is managed by Markdown mail. Add your
-	# credentials here to integrate with this.
-	# markdownmail:
-	# 	secret: ""
-	# 	list_id: ""
-	#
-	# Fill in your unique token from various analytics services to enable
-	# them.
-	# analytics:
-	# 	mixpanel:
-	# 		token: ""
-	# 	ga:
-	# 		token: ""
-	# 	heap:
-	# 		token: ""
-	# 
-	# ShareLaTeX's help desk is provided by tenderapp.com
-	# tenderUrl: ""
-	#
-
-	# Production Settings
-	# -------------------
-
-	# Should javascript assets be served minified or not. Note that you will
-	# need to run `grunt compile:minify` within the web-sharelatex directory
-	# to generate these.
-	useMinifiedJs: false
-
-	# Should static assets be sent with a header to tell the browser to cache
-	# them.
-	cacheStaticAssets: false
-
-	# If you are running ShareLaTeX over https, set this to true to send the
-	# cookie with a secure flag (recommended).
-	secureCookie: false
-
-	# Internal configs
-	# ----------------
-	path: 
-		# If we ever need to write something to disk (e.g. incoming requests
-		# that need processing but may be too big for memory, then write
-		# them to disk here).
-		dumpFolder: Path.resolve "data/dumpFolder"
-		# Where to write the project to disk before running LaTeX on it
-		compilesDir:  "/var/lib/sharelatex/compiles"
-		# Where to cache downloaded URLs for the CLSI
-		clsiCacheDir: "/var/lib/sharelatex/clsi-cache"
-	
-	# Automatic Snapshots
-	# -------------------
-	automaticSnapshots:
-		# How long should we wait after the user last edited to 
-		# take a snapshot?
-		waitTimeAfterLastEdit: 5 * minutes
-		# Even if edits are still taking place, this is maximum
-		# time to wait before taking another snapshot.
-		maxTimeBetweenSnapshots: 30 * minutes
-
-	# Smoke test
-	# ----------
-	# Provide log in credentials and a project to be able to run
-	# some basic smoke tests to check the core functionality.
-	#
-	# smokeTest:
-	# 	user: ""
-	# 	password: ""
-	# 	projectId: ""
-
-	# Filestore health check
-	# ----------------------
-	# Project and file details to check in filestore when calling /health_check
-	# health_check:
-	# 	project_id: ""
-	# 	file_id: ""

chef/cookbooks/texlive/attributes/default.rb

@@ -1,2 +1,2 @@
 default[:texlive][:schema]  = "small"
-default[:texlive][:bin_dir] = "/usr/local/texlive/2013/bin/x86_64-linux"
+default[:texlive][:bin_dir] = "/usr/local/texlive/2014/bin/x86_64-linux"

config/settings.development.coffee.example

@@ -1,275 +1,227 @@
 Path = require('path')
-http = require('http')
-http.globalAgent.maxSockets = 300
-
-# Make time interval config easier.
-seconds = 1000
-minutes = 60 * seconds
 
 # These credentials are used for authenticating api requests
 # between services that may need to go over public channels
 httpAuthUser = "sharelatex"
-httpAuthPass = "password"
+httpAuthPass = "CRYPTO_RANDOM" # Randomly generated for you
 httpAuthUsers = {}
 httpAuthUsers[httpAuthUser] = httpAuthPass
 
-sessionSecret = "secret-please-change"
+DATA_DIR = Path.resolve(Path.join(__dirname, "..", "data"))
+TMP_DIR = Path.resolve(Path.join(__dirname, "..", "tmp"))
 
 module.exports =
-	# File storage
-	# ------------
-	#
-	# ShareLaTeX needs somewhere to store binary files like images.
-	# There are currently two options:
-	#     Your local filesystem (the default)
-	#     Amazon S3
-	filestore:
-		# which backend persistor to use.
-		# choices are 
-		# s3 - Amazon S3
-		# fs - local filesystem
-		backend: "fs"	
-		stores:
-			# where to store user and template binary files
-			#
-			# For Amazon S3 this is the bucket name to store binary files
-			#
-			# For local filesystem this is the directory to store the files in.
-			# This path must exist, not be tmpfs and be writable to by the user sharelatex is run as.
-			user_files: Path.resolve(__dirname + "/../user_files")
-		# Uncomment if you need to configure your S3 credentials
-		# s3:
-		# 	# if you are using S3, then fill in your S3 details below
-		# 	key: ""
-		# 	secret: ""
-
 	# Databases
 	# ---------
+
+	# ShareLaTeX's main persistant data store is MongoDB (http://www.mongodb.org/)
+	# Documentation about the URL connection string format can be found at:
+	#
+	#    http://docs.mongodb.org/manual/reference/connection-string/
+	# 
+	# The following works out of the box with Mongo's default settings:
 	mongo:
 		url : 'mongodb://127.0.0.1/sharelatex'
 
+	# Redis is used in ShareLaTeX for high volume queries, like real-time
+	# editing, and session management.
+	#
+	# The following config will work with Redis's default settings:
 	redis:
 		web:
 			host: "localhost"
 			port: "6379"
 			password: ""
 
-		api:
-			host: "localhost"
-			port: "6379"
-			password: ""
-
-		fairy:
-			host: "localhost"
-			port: "6379"
-			password: ""
-
+	# The compile server (the clsi) uses a SQL database to cache files and
+	# meta-data. sqllite is the default, and the load is low enough that this will
+	# be fine in production (we use sqllite at sharelatex.com).
+	#
+	# If you want to configure a different database, see the Sequelize documentation
+	# for available options:
+	#
+	#    https://github.com/sequelize/sequelize/wiki/API-Reference-Sequelize#example-usage
+	#
 	mysql:
 		clsi:
 			database: "clsi"
 			username: "clsi"
 			password: ""
 			dialect: "sqlite"
-			storage: Path.resolve(__dirname + "/../db.sqlite")
+			storage: Path.join(DATA_DIR, "db.sqlite")
 
-	# Service locations
-	# -----------------
+	# File storage
+	# ------------
 
-	# Configure which ports to run each service on. Generally you
-	# can leave these as they are unless you have some other services
-	# running which conflict, or want to run the web process on port 80.
-	internal:
-		web:
-			port: webPort = 3000
-			host: "localhost"
-		documentupdater:
-			port: docUpdaterPort = 3003
-			host: "localhost"
-		clsi:
-			port: clsiPort = 3013
-			host: "localhost"
-		filestore:
-			port: filestorePort = 3009
-			host: "localhost"
-		trackchanges:
-			port: trackchangesPort = 3015
-			host: "localhost"
-		docstore:
-			port: docstorePort = 3016
-			host: "localhost"
-
-	# Tell each service where to find the other services. If everything
-	# is running locally then this is easy, but they exist as separate config
-	# options incase you want to run some services on remote hosts.
-	apis:
-		web:
-			url: "http://localhost:#{webPort}"
-			user: httpAuthUser
-			pass: httpAuthPass
-		documentupdater:
-			url : "http://localhost:#{docUpdaterPort}"
-		clsi:
-			url: "http://localhost:#{clsiPort}"
-		filestore:
-			url: "http://localhost:#{filestorePort}"
-		trackchanges:
-			url: "http://localhost:#{trackchangesPort}"
-		docstore:
-			url: "http://localhost:#{docstorePort}"
-		thirdPartyDataStore:
-			url : "http://localhost:3002"
-			emptyProjectFlushDelayMiliseconds: 5 * seconds
-		tags:
-			url :"http://localhost:3012"
-		spelling:
-			url : "http://localhost:3005"
-		versioning:
-			snapshotwaitms:3000
-			url: "http://localhost:4000"
-			username: httpAuthUser
-			password: httpAuthPass
-		recurly:
-			privateKey: ""
-			apiKey: ""
-			subdomain: ""
-		chat:
-			url: "http://localhost:3010"
-		templates:
-			port: 3007
-		blog:
-			port: 3008
-		templates_api:
-			url: "http://localhost:3007"
-
-	# Where your instance of ShareLaTeX can be found publically. Used in emails
-	# that are sent out, generated links, etc.
-	siteUrl : 'http://localhost:3000'
-
-	# Same, but with http auth credentials.
-	httpAuthSiteUrl: 'http://#{httpAuthUser}:#{httpAuthPass}@localhost:3000'
-
-	# Security
-	# --------
-	security:
-		sessionSecret: sessionSecret
-
-	httpAuthUsers: httpAuthUsers
-
-	# Default features
-	# ----------------
+	# ShareLaTeX can store binary files like images either locally or in Amazon
+	# S3. The default is locally:
+	filestore:
+		backend: "fs"	
+		stores:
+			user_files: Path.join(DATA_DIR, "user_files")
+			
+	# To use Amazon S3 as a storage backend, comment out the above config, and
+	# uncomment the following, filling in your key, secret, and bucket name:
 	#
-	# You can select the features that are enabled by default for new
-	# new users.
-	defaultFeatures: defaultFeatures =
-		collaborators: -1
-		dropbox: true
-		versioning: true
+	# filestore:
+	# 	backend: "s3"
+	# 	stores:
+	# 		user_files: "BUCKET_NAME"
+	# 	s3:
+	# 		key: "AWS_KEY"
+	# 		secret: "AWS_SECRET"
+	# 		
 
-	plans: plans = [{
-		planCode: "personal"
-		name: "Personal"
-		price: 0
-		features: defaultFeatures
-	}]
-
-	# Spelling languages
+	# Local disk caching
 	# ------------------
-	#
-	# You must have the corresponding aspell package installed to 
-	# be able to use a language.
-	languages: [
-		{name: "English", code: "en"}
-	]
+	path:
+		# If we ever need to write something to disk (e.g. incoming requests
+		# that need processing but may be too big for memory), then write
+		# them to disk here:
+		dumpFolder:   Path.join(TMP_DIR, "dumpFolder")
+		# Where to write uploads before they are processed
+		uploadFolder: Path.join(TMP_DIR, "uploads")
+		# Where to write the project to disk before running LaTeX on it
+		compilesDir:  Path.join(DATA_DIR, "compiles")
+		# Where to cache downloaded URLs for the CLSI
+		clsiCacheDir: Path.join(DATA_DIR, "cache")
 
-	# Email support
+	# Server Config
 	# -------------
-	#
-	# ShareLaTeX uses nodemailer (http://www.nodemailer.com/) to send transactional emails.
-	# To see the range of transport and options they support, see http://www.nodemailer.com/docs/transports
-	# email:
-	#	fromAddress: ""
-	#	replyTo: ""
-	#	lifecycle: false
-	#	transport: "SES"
-	#	parameters:
-	#		AWSAccessKeyID: ""
-	#		AWSSecretKey: ""
 
+	# Where your instance of ShareLaTeX can be found publicly. This is used
+	# when emails are sent out and in generated links:
+	siteUrl : 'http://localhost:3000'
+	
+	# The websocket layer of ShareLaTeX runs as separate service.
+	# When running locally or in development, you can point the client to this
+	# service directly on port 3026. If you are running behind a reverse proxy (Nginx, etc)
+	# then websocketsUrl should be the same as siteUrl, with your reverse
+	# proxy responible for sending websocket traffic to the websocket service
+	# rather than connecting directly.
+	websocketsUrl: 'http://localhost:3026'
+	
+	# If provided, a sessionSecret is used to sign cookies so that they cannot be
+	# spoofed. This is recommended.
+	security:
+		sessionSecret: "CRYPTO_RANDOM" # This was randomly generated for you
 
-	# Third party services
-	# --------------------
-	#
-	# ShareLaTeX's regular newsletter is managed by Markdown mail. Add your
-	# credentials here to integrate with this.
-	# markdownmail:
-	# 	secret: ""
-	# 	list_id: ""
-	#
-	# Fill in your unique token from various analytics services to enable
-	# them.
-	# analytics:
-	# 	mixpanel:
-	# 		token: ""
-	# 	ga:
-	# 		token: ""
-	# 	heap:
-	# 		token: ""
-	# 
-	# ShareLaTeX's help desk is provided by tenderapp.com
-	# tenderUrl: ""
-	#
-
-	# Production Settings
-	# -------------------
-
+	# These credentials are used for authenticating api requests
+	# between services that may need to go over public channels
+	httpAuthUsers: httpAuthUsers
+	
 	# Should javascript assets be served minified or not. Note that you will
 	# need to run `grunt compile:minify` within the web-sharelatex directory
 	# to generate these.
 	useMinifiedJs: false
 
 	# Should static assets be sent with a header to tell the browser to cache
-	# them.
+	# them. This should be false in development where changes are being made,
+	# but should be set to true in production.
 	cacheStaticAssets: false
 
 	# If you are running ShareLaTeX over https, set this to true to send the
 	# cookie with a secure flag (recommended).
 	secureCookie: false
-
-	# Internal configs
-	# ----------------
-	path:
-		# If we ever need to write something to disk (e.g. incoming requests
-		# that need processing but may be too big for memory, then write
-		# them to disk here).
-		dumpFolder: Path.resolve "data/dumpFolder"
-		# Where to write the project to disk before running LaTeX on it
-		compilesDir:  Path.resolve(__dirname + "/../compiles")
-		# Where to cache downloaded URLs for the CLSI
-		clsiCacheDir: Path.resolve(__dirname + "/../cache")
 	
-	# Automatic Snapshots
-	# -------------------
-	automaticSnapshots:
-		# How long should we wait after the user last edited to 
-		# take a snapshot?
-		waitTimeAfterLastEdit: 5 * minutes
-		# Even if edits are still taking place, this is maximum
-		# time to wait before taking another snapshot.
-		maxTimeBetweenSnapshots: 30 * minutes
+	# If you are running ShareLaTeX behind a proxy (like Apache, Nginx, etc)
+	# then set this to true to allow it to correctly detect the forwarded IP
+	# address and http/https protocol information.
+	behindProxy: false
 
-	# Smoke test
-	# ----------
-	# Provide log in credentials and a project to be able to run
-	# some basic smoke tests to check the core functionality.
+	# Sending Email
+	# -------------
 	#
-	# smokeTest:
-	# 	user: ""
-	# 	password: ""
-	# 	projectId: ""
+	# You must configure a mail server to be able to send invite emails from
+	# ShareLaTeX. The config settings are passed to nodemailer. See the nodemailer
+	# documentation for available options:
+	#
+	#     http://www.nodemailer.com/docs/transports
+	#
+	# email:
+	#	fromAddress: ""
+	#	replyTo: ""
+	#	transport: "SES"
+	#	parameters:
+	#		AWSAccessKeyID: ""
+	#		AWSSecretKey: ""
 
-	# Filestore health check
-	# ----------------------
-	# Project and file details to check in filestore when calling /health_check
-	# health_check:
-	# 	project_id: ""
-	# 	file_id: ""
+	# Spell Check Languages
+	# ---------------------
+	#
+	# You must have the corresponding aspell dictionary installed to 
+	# be able to use a language. Run `grunt check:aspell` to check which
+	# dictionaries you have installed. These should be set for the `code` for
+	# each language.
+	languages: [
+		{name: "English", code: "en"}
+	]
+	
+	# Service locations
+	# -----------------
+
+	# ShareLaTeX is comprised of many small services, which each expose
+	# an HTTP API running on a different port. Generally you
+	# can leave these as they are unless you have some other services
+	# running which conflict, or want to run the web process on port 80.
+	# internal:
+	# 	web:
+	# 		port: webPort = 3000
+	# 		host: "localhost"
+	# 	documentupdater:
+	# 		port: docUpdaterPort = 3003
+	# 		host: "localhost"
+	# 	filestore:
+	# 		port: filestorePort = 3009
+	# 		host: "localhost"
+	# 	chat:
+	# 		port: chatPort = 3010
+	# 		host: "localhost"
+	# 	tags:
+	# 		port: tagsPort = 3012
+	# 		host: "localhost"
+	# 	clsi:
+	# 		port: clsiPort = 3013
+	# 		host: "localhost"
+	# 	trackchanges:
+	# 		port: trackchangesPort = 3015
+	# 		host: "localhost"
+	# 	docstore:
+	# 		port: docstorePort = 3016
+	# 		host: "localhost"
+	# 	spelling:
+	# 		port: spellingPort = 3005
+	# 		host: "localhost"
+
+	# If you change the above config, or run some services on remote servers,
+	# you need to tell the other services where to find them:
+	apis:
+		web:
+			url: "http://localhost:3000"
+			user: httpAuthUser
+			pass: httpAuthPass
+	# 	documentupdater:
+	# 		url : "http://localhost:#{docUpdaterPort}"
+	# 	clsi:
+	# 		url: "http://localhost:#{clsiPort}"
+	# 	filestore:
+	# 		url: "http://localhost:#{filestorePort}"
+	# 	trackchanges:
+	# 		url: "http://localhost:#{trackchangesPort}"
+	# 	docstore:
+	# 		url: "http://localhost:#{docstorePort}"
+	# 	tags:
+	# 		url: "http://localhost:#{tagsPort}"
+	# 	spelling:
+	# 		url: "http://localhost:#{spellingPort}"
+	# 	chat:
+	# 		url: "http://localhost:#{chatPort}"
+	
+
+# With lots of incoming and outgoing HTTP connections to different services,
+# sometimes long running, it is a good idea to increase the default number
+# of sockets that Node will hold open.
+http = require('http')
+http.globalAgent.maxSockets = 300
+https = require('https')
+https.globalAgent.maxSockets = 300

migrations/1_doc_lines.coffee

@@ -0,0 +1,161 @@
+Settings = require "settings-sharelatex"
+fs = require("fs")
+mongojs = require("mongojs")
+ObjectId = mongojs.ObjectId
+console.log Settings.mongo.url
+db = mongojs(Settings.mongo.url, ['projects', 'docs'])
+_ = require("lodash")
+async = require("async")
+exec = require("child_process").exec
+
+finished_projects_path = "/tmp/finished-projects"
+all_projects_path = "/tmp/all-projects"
+project_too_large_path = "/tmp/large_projects"
+
+
+printProgress = ->
+	exec "wc #{finished_projects_path}", (error, results) ->
+		setTimeout printProgress, 1000 * 30
+
+checkIfFileHasBeenProccessed = (project_id, callback)->
+	exec "grep #{project_id} #{finished_projects_path}", (error, results) ->
+		hasBeenProcessed = _.include(results, project_id)
+		callback(error, hasBeenProcessed)
+
+loadProjectIds = (callback)->
+	console.log "loading project ids from #{all_projects_path}"
+	fs.readFile all_projects_path, "utf-8", (err, data)->
+		ids = data.split("\n")
+		console.log "loaded #{ids.length} project ids from #{all_projects_path}"
+		callback err, ids
+
+getAndWriteProjectids = (callback)->
+	console.log "finding all project id's - #{new Date().toString()}"
+	db.projects.find {}, {_id:1}, (err, ids)->
+		console.log "total found projects in mongo #{ids.length} - #{new Date().toString()}"
+		ids = _.pluck ids, '_id'
+		ids = _.filter ids, (id)-> id?
+		fileData = ids.join("\n")
+		fs.writeFile all_projects_path, fileData, ->
+			callback(err, ids)
+
+markProjectAsToLargeAndFinished = (project_id, callback)->
+	console.log "#{project_id} too large"
+	markProjectAsProcessed project_id, (err)->
+		fs.appendFile project_too_large_path, "#{project_id}\n", callback
+
+getProjectIds = (callback)->
+	exists = fs.existsSync all_projects_path
+	if exists
+		loadProjectIds callback
+	else
+		getAndWriteProjectids callback
+
+markProjectAsProcessed = (project_id, callback)->
+	fs.appendFile finished_projects_path, "#{project_id}\n", callback
+
+getAllDocs = (project_id, callback = (error, docs) ->) ->
+	db.projects.findOne _id:ObjectId(project_id), (error, project) ->
+		return callback(error) if error?
+		if !project?
+			console.log "no such project #{project_id}"
+			return callback()
+		size = require("../node_modules/mongojs/node_modules/mongodb/node_modules/bson/").BSONPure.BSON.calculateObjectSize(project)
+		if size > 12000000 #12mb
+			return markProjectAsToLargeAndFinished project_id, callback
+		findAllDocsInProject project, (error, docs) ->
+			return callback(error) if error?
+			return callback null, docs
+
+findAllDocsInProject = (project, callback = (error, docs) ->) ->
+	callback null, _findAllDocsInFolder project.rootFolder[0]
+
+_findDocInFolder = (folder = {}, doc_id, currentPath) ->
+	for doc, i in folder.docs or []
+		if doc?._id? and doc._id.toString() == doc_id.toString()
+			return {
+				doc: doc
+				mongoPath: "#{currentPath}.docs.#{i}"
+			}
+
+	for childFolder, i in folder.folders or []
+		result = _findDocInFolder childFolder, doc_id, "#{currentPath}.folders.#{i}"
+		return result if result?
+
+	return null
+
+_findAllDocsInFolder = (folder = {}) ->
+	docs = folder.docs or []
+	for childFolder in folder.folders or []
+		docs = docs.concat _findAllDocsInFolder childFolder
+	return docs
+
+insertDocIntoDocCollection = (project_id, doc_id, lines, oldRev, callback)->
+	if !project_id?
+		return callback("no project id")
+	if !doc_id?
+		return callback()
+	if !lines?
+		lines = [""]
+	update = {}
+	update["_id"] = ObjectId(doc_id.toString())
+	update["lines"] = lines
+	update["project_id"] = ObjectId(project_id)
+	update["rev"] = oldRev || 0
+	db.docs.insert update, callback
+
+saveDocsIntoMongo = (project_id, docs, callback)->
+	jobs = _.map docs, (doc)->
+		(cb)->
+			if !doc?
+				console.error "null doc in project #{project_id}" #just skip it, not a big deal
+				return cb()
+			insertDocIntoDocCollection project_id, doc._id, doc.lines, doc.rev, (err)->
+				if err?.code == 11000 #duplicate key, doc already in there so its not a problem.
+					err = undefined
+				if err?
+					console.log "error inserting doc into doc collection", err
+				cb(err)
+
+
+	async.series jobs, callback
+
+
+processNext = (project_id, callback)->
+	checkIfFileHasBeenProccessed project_id, (err, hasBeenProcessed)->
+		if hasBeenProcessed
+			console.log "#{project_id} already procssed, skipping"
+			return callback()
+		console.log "#{project_id} processing"
+		getAllDocs project_id, (err, docs)->
+			if err?
+				console.error err, project_id, "could not get all docs"
+				return callback(err)
+			else
+				saveDocsIntoMongo project_id, docs, (err)->
+					if err?
+						console.error err, project_id, "could not save docs into mongo"
+						return callback(err)
+					markProjectAsProcessed project_id, (err)->
+						setTimeout(
+							-> callback(err)
+						,500)
+
+
+
+exports.migrate = (client, done = ->)->
+	getProjectIds (err, ids)->
+		printProgress()
+		jobs = _.map ids, (id)->
+			return (cb)->
+				processNext(id, cb)
+		async.series jobs, (err)->
+			if err?
+				console.error err, "at end of jobs"
+			else
+				console.log "finished"
+			done(err)
+
+
+exports.rollback = (next)->
+	next()

migrations/about_migrations.md

@@ -0,0 +1,2 @@
+* if migration is stopped mid way it will start at the beginging next time
+* to see the run migrations do db.getCollection('_migrations').find() you can't do db._migrations.find()

package.json

@@ -1,10 +1,17 @@
 {
   "name": "sharelatex",
-  "version": "0.0.1",
+  "version": "0.1.3",
   "description": "An online collaborative LaTeX editor",
   "dependencies": {
+    "async": "^0.9.0",
+    "east": "^0.2.3",
+    "east-mongo": "^0.1.2",
+    "grunt-shell": "^1.1.1",
+    "lodash": "^3.0.0",
+    "mongojs": "^0.18.1",
     "rimraf": "~2.2.6",
-    "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git"
+    "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git",
+    "underscore": "^1.7.0"
   },
   "devDependencies": {
     "grunt": "~0.4.2",
@@ -13,6 +20,7 @@
     "grunt-execute": "~0.1.5",
     "grunt-available-tasks": "~0.4.1",
     "grunt-concurrent": "~0.4.3",
+    "grunt-contrib-coffee": "~0.10.1",
     "semver": "~2.2.1",
     "knox": "~0.8.9"
   }

package/config/settings.coffee

@@ -0,0 +1,219 @@
+Path = require('path')
+
+# These credentials are used for authenticating api requests
+# between services that may need to go over public channels
+httpAuthUser = "sharelatex"
+httpAuthPass = "CRYPTO_RANDOM" # Randomly generated for you
+httpAuthUsers = {}
+httpAuthUsers[httpAuthUser] = httpAuthPass
+
+DATA_DIR = '/var/lib/sharelatex/data'
+TMP_DIR = '/var/lib/sharelatex/tmp'
+
+module.exports =
+	# Databases
+	# ---------
+
+	# ShareLaTeX's main persistant data store is MongoDB (http://www.mongodb.org/)
+	# Documentation about the URL connection string format can be found at:
+	#
+	#    http://docs.mongodb.org/manual/reference/connection-string/
+	# 
+	# The following works out of the box with Mongo's default settings:
+	mongo:
+		url : 'mongodb://127.0.0.1/sharelatex'
+
+	# Redis is used in ShareLaTeX for high volume queries, like real-time
+	# editing, and session management.
+	#
+	# The following config will work with Redis's default settings:
+	redis:
+		web:
+			host: "localhost"
+			port: "6379"
+			password: ""
+
+	# The compile server (the clsi) uses a SQL database to cache files and
+	# meta-data. sqllite is the default, and the load is low enough that this will
+	# be fine in production (we use sqllite at sharelatex.com).
+	#
+	# If you want to configure a different database, see the Sequelize documentation
+	# for available options:
+	#
+	#    https://github.com/sequelize/sequelize/wiki/API-Reference-Sequelize#example-usage
+	#
+	mysql:
+		clsi:
+			database: "clsi"
+			username: "clsi"
+			password: ""
+			dialect: "sqlite"
+			storage: Path.join(DATA_DIR, "db.sqlite")
+
+	# File storage
+	# ------------
+
+	# ShareLaTeX can store binary files like images either locally or in Amazon
+	# S3. The default is locally:
+	filestore:
+		backend: "fs"	
+		stores:
+			user_files: Path.join(DATA_DIR, "user_files")
+			
+	# To use Amazon S3 as a storage backend, comment out the above config, and
+	# uncomment the following, filling in your key, secret, and bucket name:
+	#
+	# filestore:
+	# 	backend: "s3"
+	# 	stores:
+	# 		user_files: "BUCKET_NAME"
+	# 	s3:
+	# 		key: "AWS_KEY"
+	# 		secret: "AWS_SECRET"
+	# 		
+
+	# Local disk caching
+	# ------------------
+	path:
+		# If we ever need to write something to disk (e.g. incoming requests
+		# that need processing but may be too big for memory), then write
+		# them to disk here:
+		dumpFolder:   Path.join(TMP_DIR, "dumpFolder")
+		# Where to write uploads before they are processed
+		uploadFolder: Path.join(TMP_DIR, "uploads")
+		# Where to write the project to disk before running LaTeX on it
+		compilesDir:  Path.join(DATA_DIR, "compiles")
+		# Where to cache downloaded URLs for the CLSI
+		clsiCacheDir: Path.join(DATA_DIR, "cache")
+
+	# Server Config
+	# -------------
+
+	# Where your instance of ShareLaTeX can be found publicly. This is used
+	# when emails are sent out and in generated links:
+	siteUrl : 'http://localhost:3000'
+	
+	# If provided, a sessionSecret is used to sign cookies so that they cannot be
+	# spoofed. This is recommended.
+	security:
+		sessionSecret: "CRYPTO_RANDOM" # This was randomly generated for you
+
+	# These credentials are used for authenticating api requests
+	# between services that may need to go over public channels
+	httpAuthUsers: httpAuthUsers
+	
+	# Should javascript assets be served minified or not. Note that you will
+	# need to run `grunt compile:minify` within the web-sharelatex directory
+	# to generate these.
+	useMinifiedJs: false
+
+	# Should static assets be sent with a header to tell the browser to cache
+	# them. This should be false in development where changes are being made,
+	# but should be set to true in production.
+	cacheStaticAssets: false
+
+	# If you are running ShareLaTeX over https, set this to true to send the
+	# cookie with a secure flag (recommended).
+	secureCookie: false
+	
+	# If you are running ShareLaTeX behind a proxy (like Apache, Nginx, etc)
+	# then set this to true to allow it to correctly detect the forwarded IP
+	# address and http/https protocol information.
+	behindProxy: false
+
+	# Sending Email
+	# -------------
+	#
+	# You must configure a mail server to be able to send invite emails from
+	# ShareLaTeX. The config settings are passed to nodemailer. See the nodemailer
+	# documentation for available options:
+	#
+	#     http://www.nodemailer.com/docs/transports
+	#
+	# email:
+	#	fromAddress: ""
+	#	replyTo: ""
+	#	transport: "SES"
+	#	parameters:
+	#		AWSAccessKeyID: ""
+	#		AWSSecretKey: ""
+
+	# Spell Check Languages
+	# ---------------------
+	#
+	# You must have the corresponding aspell dictionary installed to 
+	# be able to use a language. Run `grunt check:aspell` to check which
+	# dictionaries you have installed. These should be set for the `code` for
+	# each language.
+	languages: [
+		{name: "English", code: "en"}
+	]
+	
+	# Service locations
+	# -----------------
+
+	# ShareLaTeX is comprised of many small services, which each expose
+	# an HTTP API running on a different port. Generally you
+	# can leave these as they are unless you have some other services
+	# running which conflict, or want to run the web process on port 80.
+	# internal:
+	# 	web:
+	# 		port: webPort = 3000
+	# 		host: "localhost"
+	# 	documentupdater:
+	# 		port: docUpdaterPort = 3003
+	# 		host: "localhost"
+	# 	filestore:
+	# 		port: filestorePort = 3009
+	# 		host: "localhost"
+	# 	chat:
+	# 		port: chatPort = 3010
+	# 		host: "localhost"
+	# 	tags:
+	# 		port: tagsPort = 3012
+	# 		host: "localhost"
+	# 	clsi:
+	# 		port: clsiPort = 3013
+	# 		host: "localhost"
+	# 	trackchanges:
+	# 		port: trackchangesPort = 3015
+	# 		host: "localhost"
+	# 	docstore:
+	# 		port: docstorePort = 3016
+	# 		host: "localhost"
+	# 	spelling:
+	# 		port: spellingPort = 3005
+	# 		host: "localhost"
+
+	# If you change the above config, or run some services on remote servers,
+	# you need to tell the other services where to find them:
+	apis:
+		web:
+			url: "http://localhost:3000"
+			user: httpAuthUser
+			pass: httpAuthPass
+	# 	documentupdater:
+	# 		url : "http://localhost:#{docUpdaterPort}"
+	# 	clsi:
+	# 		url: "http://localhost:#{clsiPort}"
+	# 	filestore:
+	# 		url: "http://localhost:#{filestorePort}"
+	# 	trackchanges:
+	# 		url: "http://localhost:#{trackchangesPort}"
+	# 	docstore:
+	# 		url: "http://localhost:#{docstorePort}"
+	# 	tags:
+	# 		url: "http://localhost:#{tagsPort}"
+	# 	spelling:
+	# 		url: "http://localhost:#{spellingPort}"
+	# 	chat:
+	# 		url: "http://localhost:#{chatPort}"
+	
+
+# With lots of incoming and outgoing HTTP connections to different services,
+# sometimes long running, it is a good idea to increase the default number
+# of sockets that Node will hold open.
+http = require('http')
+http.globalAgent.maxSockets = 300
+https = require('https')
+https.globalAgent.maxSockets = 300

package/nginx/sharelatex

@@ -0,0 +1,32 @@
+server {
+	listen         80;
+	server_name    _; # Catch all, see http://nginx.org/en/docs/http/server_names.html
+
+	set $static_path /var/www/sharelatex/web/public;
+
+	location / {
+		proxy_pass http://localhost:3000;
+		proxy_set_header Host $http_x_forwarded_host;
+		proxy_http_version 1.1;
+		proxy_set_header Upgrade $http_upgrade;
+		proxy_set_header Connection "upgrade";
+		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+		proxy_read_timeout 3m;
+		proxy_send_timeout 3m;
+	}
+
+	location /stylesheets {
+		expires 1y;
+		root $static_path/;
+	}
+
+	location /minjs {
+		expires 1y;
+		root $static_path/;
+	}
+
+	location /img {
+		expires 1y;
+		root $static_path/;
+	}
+}

package/scripts/after_install.sh

@@ -1,21 +1,26 @@
 #!/bin/sh
+# Create random secret keys (twice, once for http auth pass, once for cookie secret).
+sed -i "0,/CRYPTO_RANDOM/s/CRYPTO_RANDOM/$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 64 | head -n 1)/" /etc/sharelatex/settings.coffee
+sed -i "0,/CRYPTO_RANDOM/s/CRYPTO_RANDOM/$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 64 | head -n 1)/" /etc/sharelatex/settings.coffee
+
 sudo adduser --system --group --home /var/www/sharelatex --no-create-home sharelatex
 
 mkdir -p /var/log/sharelatex
 chown sharelatex:sharelatex /var/log/sharelatex
-mkdir -p /var/data/sharelatex/user_files
-chown sharelatex:sharelatex /var/data/sharelatex/user_files
-mkdir -p /var/data/sharelatex/uploads
-chown sharelatex:sharelatex /var/data/sharelatex/uploads
-mkdir -p /var/data/sharelatex/compiles
-chown sharelatex:sharelatex /var/data/sharelatex/compiles
-mkdir -p /var/data/sharelatex/cache
-chown sharelatex:sharelatex /var/data/sharelatex/cache
-mkdir -p /var/data/sharelatex/dump
-chown sharelatex:sharelatex /var/data/sharelatex/dump
+
+mkdir -p /var/lib/sharelatex
+mkdir -p /var/lib/sharelatex/data/user_files
+mkdir -p /var/lib/sharelatex/tmp/uploads
+mkdir -p /var/lib/sharelatex/data/compiles
+mkdir -p /var/lib/sharelatex/data/cache
+mkdir -p /var/lib/sharelatex/tmp/dumpFolder
+chown -R sharelatex:sharelatex /var/lib/sharelatex
 service sharelatex-web restart
 service sharelatex-document-updater restart
 service sharelatex-clsi restart
 service sharelatex-filestore restart
 service sharelatex-track-changes restart
 service sharelatex-docstore restart
+service sharelatex-chat restart
+service sharelatex-tags restart
+service sharelatex-spelling restart

package/upstart/sharelatex-chat

@@ -0,0 +1,28 @@
+description "sharelatex-web"
+author      "ShareLaTeX <team@sharelatex.com>"
+
+start on (local-filesystems and net-device-up IFACE!=lo)
+stop on shutdown
+
+respawn
+
+limit nofile 8192 8192
+
+pre-start script
+    mkdir -p /var/log/sharelatex
+end script
+
+script
+    SERVICE=chat
+    USER=sharelatex
+    GROUP=sharelatex
+    # You may need to replace this with an absolute 
+    # path to Node.js if it's not in your system PATH.
+    NODE=node
+    SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee
+    LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
+
+    echo $$ > /var/run/sharelatex-$SERVICE.pid
+    cd /var/www/sharelatex/$SERVICE
+    exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
+end script

package/upstart/sharelatex-clsi

@@ -1,7 +1,7 @@
-description "sharelatex-clsi"
+description "sharelatex-web"
 author      "ShareLaTeX <team@sharelatex.com>"
 
-start on started mountall
+start on (local-filesystems and net-device-up IFACE!=lo)
 stop on shutdown
 
 respawn
@@ -9,11 +9,20 @@ respawn
 limit nofile 8192 8192
 
 pre-start script
-	mkdir -p /var/log/sharelatex
+    mkdir -p /var/log/sharelatex
 end script
 
 script
-	echo $$ > /var/run/sharelatex-clsi.pid
-	chdir /var/www/sharelatex/clsi
-	exec sudo -u sharelatex -g sharelatex env SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee NODE_ENV=production node app.js >> /var/log/sharelatex/clsi.log 2>&1
-end script
+    SERVICE=clsi
+    USER=sharelatex
+    GROUP=sharelatex
+    # You may need to replace this with an absolute 
+    # path to Node.js if it's not in your system PATH.
+    NODE=node
+    SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee
+    LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
+
+    echo $$ > /var/run/sharelatex-$SERVICE.pid
+    cd /var/www/sharelatex/$SERVICE
+    exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
+end script

package/upstart/sharelatex-docstore

@@ -1,7 +1,7 @@
-description "sharelatex-docstore"
+description "sharelatex-web"
 author      "ShareLaTeX <team@sharelatex.com>"
 
-start on started mountall
+start on (local-filesystems and net-device-up IFACE!=lo)
 stop on shutdown
 
 respawn
@@ -9,11 +9,20 @@ respawn
 limit nofile 8192 8192
 
 pre-start script
-	mkdir -p /var/log/sharelatex
+    mkdir -p /var/log/sharelatex
 end script
 
 script
-	echo $$ > /var/run/sharelatex-docstore.pid
-	chdir /var/www/sharelatex/docstore
-	exec sudo -u sharelatex -g sharelatex env SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee NODE_ENV=production node app.js >> /var/log/sharelatex/docstore.log 2>&1
-end script
+    SERVICE=docstore
+    USER=sharelatex
+    GROUP=sharelatex
+    # You may need to replace this with an absolute 
+    # path to Node.js if it's not in your system PATH.
+    NODE=node
+    SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee
+    LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
+
+    echo $$ > /var/run/sharelatex-$SERVICE.pid
+    cd /var/www/sharelatex/$SERVICE
+    exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
+end script

package/upstart/sharelatex-document-updater

@@ -1,7 +1,7 @@
-description "sharelatex-document-updater"
+description "sharelatex-web"
 author      "ShareLaTeX <team@sharelatex.com>"
 
-start on started mountall
+start on (local-filesystems and net-device-up IFACE!=lo)
 stop on shutdown
 
 respawn
@@ -9,11 +9,20 @@ respawn
 limit nofile 8192 8192
 
 pre-start script
-	mkdir -p /var/log/sharelatex
+    mkdir -p /var/log/sharelatex
 end script
 
 script
-	echo $$ > /var/run/sharelatex-document-updater.pid
-	chdir /var/www/sharelatex/document-updater
-	exec sudo -u sharelatex -g sharelatex env SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee NODE_ENV=production node app.js >> /var/log/sharelatex/document-updater.log 2>&1
-end script
+    SERVICE=document-updater
+    USER=sharelatex
+    GROUP=sharelatex
+    # You may need to replace this with an absolute 
+    # path to Node.js if it's not in your system PATH.
+    NODE=node
+    SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee
+    LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
+
+    echo $$ > /var/run/sharelatex-$SERVICE.pid
+    cd /var/www/sharelatex/$SERVICE
+    exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
+end script

package/upstart/sharelatex-filestore

@@ -1,7 +1,7 @@
-description "sharelatex-filestore"
+description "sharelatex-web"
 author      "ShareLaTeX <team@sharelatex.com>"
 
-start on started mountall
+start on (local-filesystems and net-device-up IFACE!=lo)
 stop on shutdown
 
 respawn
@@ -9,11 +9,20 @@ respawn
 limit nofile 8192 8192
 
 pre-start script
-	mkdir -p /var/log/sharelatex
+    mkdir -p /var/log/sharelatex
 end script
 
 script
-	echo $$ > /var/run/sharelatex-filestore.pid
-	chdir /var/www/sharelatex/filestore
-	exec sudo -u sharelatex -g sharelatex env SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee NODE_ENV=production node app.js >> /var/log/sharelatex/filestore.log 2>&1
-end script
+    SERVICE=filestore
+    USER=sharelatex
+    GROUP=sharelatex
+    # You may need to replace this with an absolute 
+    # path to Node.js if it's not in your system PATH.
+    NODE=node
+    SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee
+    LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
+
+    echo $$ > /var/run/sharelatex-$SERVICE.pid
+    cd /var/www/sharelatex/$SERVICE
+    exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
+end script

package/upstart/sharelatex-real-time

@@ -0,0 +1,28 @@
+description "sharelatex-web"
+author      "ShareLaTeX <team@sharelatex.com>"
+
+start on (local-filesystems and net-device-up IFACE!=lo)
+stop on shutdown
+
+respawn
+
+limit nofile 8192 8192
+
+pre-start script
+    mkdir -p /var/log/sharelatex
+end script
+
+script
+    SERVICE=real-time
+    USER=sharelatex
+    GROUP=sharelatex
+    # You may need to replace this with an absolute 
+    # path to Node.js if it's not in your system PATH.
+    NODE=node
+    SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee
+    LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
+
+    echo $$ > /var/run/sharelatex-$SERVICE.pid
+    cd /var/www/sharelatex/$SERVICE
+    exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
+end script

package/upstart/sharelatex-spelling

@@ -0,0 +1,28 @@
+description "sharelatex-web"
+author      "ShareLaTeX <team@sharelatex.com>"
+
+start on (local-filesystems and net-device-up IFACE!=lo)
+stop on shutdown
+
+respawn
+
+limit nofile 8192 8192
+
+pre-start script
+    mkdir -p /var/log/sharelatex
+end script
+
+script
+    SERVICE=spelling
+    USER=sharelatex
+    GROUP=sharelatex
+    # You may need to replace this with an absolute 
+    # path to Node.js if it's not in your system PATH.
+    NODE=node
+    SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee
+    LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
+
+    echo $$ > /var/run/sharelatex-$SERVICE.pid
+    cd /var/www/sharelatex/$SERVICE
+    exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
+end script

package/upstart/sharelatex-tags

@@ -0,0 +1,28 @@
+description "sharelatex-web"
+author      "ShareLaTeX <team@sharelatex.com>"
+
+start on (local-filesystems and net-device-up IFACE!=lo)
+stop on shutdown
+
+respawn
+
+limit nofile 8192 8192
+
+pre-start script
+    mkdir -p /var/log/sharelatex
+end script
+
+script
+    SERVICE=tags
+    USER=sharelatex
+    GROUP=sharelatex
+    # You may need to replace this with an absolute 
+    # path to Node.js if it's not in your system PATH.
+    NODE=node
+    SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee
+    LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
+
+    echo $$ > /var/run/sharelatex-$SERVICE.pid
+    cd /var/www/sharelatex/$SERVICE
+    exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
+end script

package/upstart/sharelatex-template

@@ -1,7 +1,7 @@
-description "sharelatex-SERVICE"
+description "sharelatex-web"
 author      "ShareLaTeX <team@sharelatex.com>"
 
-start on started mountall
+start on (local-filesystems and net-device-up IFACE!=lo)
 stop on shutdown
 
 respawn
@@ -9,11 +9,20 @@ respawn
 limit nofile 8192 8192
 
 pre-start script
-	mkdir -p /var/log/sharelatex
+    mkdir -p /var/log/sharelatex
 end script
 
 script
-	echo $$ > /var/run/sharelatex-SERVICE.pid
-	chdir /var/www/sharelatex/SERVICE
-	exec sudo -u sharelatex -g sharelatex env SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee NODE_ENV=production node app.js >> /var/log/sharelatex/SERVICE.log 2>&1
-end script
+    SERVICE=__SERVICE__
+    USER=sharelatex
+    GROUP=sharelatex
+    # You may need to replace this with an absolute 
+    # path to Node.js if it's not in your system PATH.
+    NODE=node
+    SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee
+    LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
+
+    echo $$ > /var/run/sharelatex-$SERVICE.pid
+    cd /var/www/sharelatex/$SERVICE
+    exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
+end script

package/upstart/sharelatex-track-changes

@@ -1,7 +1,7 @@
-description "sharelatex-track-changes"
+description "sharelatex-web"
 author      "ShareLaTeX <team@sharelatex.com>"
 
-start on started mountall
+start on (local-filesystems and net-device-up IFACE!=lo)
 stop on shutdown
 
 respawn
@@ -9,11 +9,20 @@ respawn
 limit nofile 8192 8192
 
 pre-start script
-	mkdir -p /var/log/sharelatex
+    mkdir -p /var/log/sharelatex
 end script
 
 script
-	echo $$ > /var/run/sharelatex-track-changes.pid
-	chdir /var/www/sharelatex/track-changes
-	exec sudo -u sharelatex -g sharelatex env SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee NODE_ENV=production node app.js >> /var/log/sharelatex/track-changes.log 2>&1
-end script
+    SERVICE=track-changes
+    USER=sharelatex
+    GROUP=sharelatex
+    # You may need to replace this with an absolute 
+    # path to Node.js if it's not in your system PATH.
+    NODE=node
+    SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee
+    LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
+
+    echo $$ > /var/run/sharelatex-$SERVICE.pid
+    cd /var/www/sharelatex/$SERVICE
+    exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
+end script

package/upstart/sharelatex-web

@@ -1,7 +1,7 @@
 description "sharelatex-web"
 author      "ShareLaTeX <team@sharelatex.com>"
 
-start on started mountall
+start on (local-filesystems and net-device-up IFACE!=lo)
 stop on shutdown
 
 respawn
@@ -9,11 +9,20 @@ respawn
 limit nofile 8192 8192
 
 pre-start script
-	mkdir -p /var/log/sharelatex
+    mkdir -p /var/log/sharelatex
 end script
 
 script
-	echo $$ > /var/run/sharelatex-web.pid
-	chdir /var/www/sharelatex/web
-	exec sudo -u sharelatex -g sharelatex env SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee NODE_ENV=production node app.js >> /var/log/sharelatex/web.log 2>&1
-end script
+    SERVICE=web
+    USER=sharelatex
+    GROUP=sharelatex
+    # You may need to replace this with an absolute 
+    # path to Node.js if it's not in your system PATH.
+    NODE=node
+    SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee
+    LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
+
+    echo $$ > /var/run/sharelatex-$SERVICE.pid
+    cd /var/www/sharelatex/$SERVICE
+    exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
+end script