Release 0.1.2

force an npm rebuild to work around issue #129, missing bcrypt bindings

.gitignore

@@ -11,6 +11,7 @@ docstore
 tags
 chat
 spelling
+real-time
 
 data
 tmp

Gruntfile.coffee

@@ -11,39 +11,43 @@ async = require "async"
 SERVICES = [{
 	name: "web"
 	repo: "https://github.com/sharelatex/web-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.2"
+}, {
+	name: "real-time"
+	repo: "https://github.com/sharelatex/real-time-sharelatex.git"
+	version: "v0.1.2"
 }, {
 	name: "document-updater"
 	repo: "https://github.com/sharelatex/document-updater-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.2"
 }, {
 	name: "clsi"
 	repo: "https://github.com/sharelatex/clsi-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.2"
 }, {
 	name: "filestore"
 	repo: "https://github.com/sharelatex/filestore-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.2"
 }, {
 	name: "track-changes"
 	repo: "https://github.com/sharelatex/track-changes-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.2"
 }, {
 	name: "docstore"
 	repo: "https://github.com/sharelatex/docstore-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.2"
 }, {
 	name: "chat"
 	repo: "https://github.com/sharelatex/chat-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.2"
 }, {
 	name: "tags"
 	repo: "https://github.com/sharelatex/tags-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.2"
 }, {
 	name: "spelling"
 	repo: "https://github.com/sharelatex/spelling-sharelatex.git"
-	version: "v0.1.0"
+	version: "v0.1.2"
 }]
 
 module.exports = (grunt) ->
@@ -51,6 +55,8 @@ module.exports = (grunt) ->
 	grunt.loadNpmTasks 'grunt-execute'
 	grunt.loadNpmTasks 'grunt-available-tasks'
 	grunt.loadNpmTasks 'grunt-concurrent'
+	grunt.loadNpmTasks "grunt-contrib-coffee"
+
 
 	execute = {}
 	for service in SERVICES
@@ -66,6 +72,18 @@ module.exports = (grunt) ->
 				options:
 					limit: SERVICES.length
 					logConcurrentOutput: true
+		coffee:
+			migrate: 
+				expand: true,
+				flatten: false,
+				cwd: './',
+				src: ['./migrations/*.coffee'],
+				dest: './',
+				ext: '.js'
+				options:
+					bare:true
+
+
 
 		availabletasks:
 			tasks:
@@ -143,24 +161,32 @@ module.exports = (grunt) ->
 	grunt.registerTask "build:upstart_scripts", "Create upstart scripts for each service", () ->
 		Helpers.buildUpstartScripts()
 
+
+	grunt.registerTask 'migrate', 'run migrations', ['coffee:migrate']
+
+
 	Helpers =
 		installService: (service, callback = (error) ->) ->
 			Helpers.cloneGitRepo service, (error) ->
 				return callback(error) if error?
 				Helpers.installNpmModules service, (error) ->
 					return callback(error) if error?
-					Helpers.runGruntInstall service, (error) ->
+					Helpers.rebuildNpmModules service, (error) ->
 						return callback(error) if error?
-						callback()
+						Helpers.runGruntInstall service, (error) ->
+							return callback(error) if error?
+							callback()
 
 		updateService: (service, callback = (error) ->) ->
 			Helpers.updateGitRepo service, (error) ->
 				return callback(error) if error?
 				Helpers.installNpmModules service, (error) ->
 					return callback(error) if error?
-					Helpers.runGruntInstall service, (error) ->
+					Helpers.rebuildNpmModules service, (error) ->
 						return callback(error) if error?
-						callback()
+						Helpers.runGruntInstall service, (error) ->
+							return callback(error) if error?
+							callback()
 
 		cloneGitRepo: (service, callback = (error) ->) ->
 			repo_src = service.repo
@@ -203,13 +229,21 @@ module.exports = (grunt) ->
 							proc = spawn "git", ["push", "--tags"], cwd: dir, stdio: "inherit"
 							proc.on "close", () ->
 								callback()
-								
+
 		installNpmModules: (service, callback = (error) ->) ->
 			dir = service.name
 			proc = spawn "npm", ["install"], stdio: "inherit", cwd: dir
 			proc.on "close", () ->
 				callback()
-				
+
+		# work around for https://github.com/npm/npm/issues/5400
+		# where binary modules are not built due to bug in npm
+		rebuildNpmModules: (service, callback = (error) ->) ->
+			dir = service.name
+			proc = spawn "npm", ["rebuild"], stdio: "inherit", cwd: dir
+			proc.on "close", () ->
+				callback()
+
 		createDataDirs: (callback = (error) ->) ->
 			DIRS = [
 				"tmp/dumpFolder"

README.md

@@ -3,6 +3,8 @@ ShareLaTeX
 
 [ShareLaTeX](https://www.sharelatex.com) is an open-source online real-time collaborative LaTeX editor. We run a hosted version at http://www.sharelatex.com, but you can also run your own local version, and contribute to the development of ShareLaTeX.
 
+*[If you want help installing and maintaining ShareLaTeX at your university or workplace, we offer an officially supported version called ShareLaTeX Server Pro. It also comes with extra security and admin features. Click here to find out more!](https://www.sharelatex.com/university/onsite.html)*
+
 Installation
 ------------
 
@@ -11,7 +13,8 @@ We have detailed installation instructions in our wiki:
 * [Installing ShareLaTeX in Production](https://github.com/sharelatex/sharelatex/wiki/Production-Installation-Instructions)
 * [Setting up a ShareLaTeX Development Environment](https://github.com/sharelatex/sharelatex/wiki/Setting-up-a-Development-Environment)
 
-**[Please help us make ShareLaTeX as easy to install as possible by answering our quick survey about your system and needs](https://sharelatex.typeform.com/to/PLNits)**
+**If you have any problems, have a look at our page of [Frequent Problems and Questions](https://github.com/sharelatex/sharelatex/wiki/FAQ).**
+
 
 Dependencies
 ------------
@@ -23,6 +26,8 @@ ShareLaTeX should run on OS X and Linux. You need:
 * A local instance of [Redis](http://redis.io/topics/quickstart) (version 2.6.12 or later) and [MongoDB](http://docs.mongodb.org/manual/installation/) running on their standard ports.
 * [TeXLive](https://www.tug.org/texlive/) 2013 or later with the `latexmk` program installed.
 
+ShareLaTeX needs a minimum of 2gb of memory, it is likely to be more than that though depending on usage.
+
 Other repositories
 ------------------
 
@@ -75,6 +80,10 @@ The backend API for managing project tags (folders).
 
 An API for running server-side spelling checking on ShareLaTeX documents.
 
+Dropbox
+-------
+
+Please note that certain features like Dropbox integration are not functional in the open source code base yet, despite appearing in the user interface. We're working on this, sorry!
 
 Contributing
 ------------

migrations/1422460849371-doc-lines.coffee

@@ -0,0 +1,133 @@
+
+fs = require("fs")
+mongojs = require("mongojs")
+ObjectId = mongojs.ObjectId
+db = mongojs('sharelatex', ['projects', 'docs'])
+_ = require("lodash")
+async = require("async")
+exec = require("child_process").exec
+
+finished_projects_path = "/tmp/finished-projects"
+all_projects_path = "/tmp/all-projects"
+
+printProgress = ->
+	exec "wc #{finished_projects_path}", (error, results) ->
+		#console.log results
+		setTimeout printProgress, 1000 * 30
+
+checkIfFileHasBeenProccessed = (project_id, callback)->
+	exec "grep #{project_id} #{finished_projects_path}", (error, results) ->
+		hasBeenProcessed = _.include(results, project_id)
+		#console.log hasBeenProcessed, project_id
+		callback(null, hasBeenProcessed)
+
+loadProjectIds = (callback)->
+	fs.readFile all_projects_path, "utf-8", (err, data)->
+		console.log data.length
+		ids = data.split("\n")
+		console.log ids.length
+		callback err, ids
+
+getAndWriteProjectids = (callback)->
+	console.log "finding all project id's - #{new Date().toString()}"
+	db.projects.find {}, {_id:1}, (err, ids)->
+		console.log "total found projects in mongo #{ids.length} - #{new Date().toString()}"
+		ids = _.pluck ids, '_id'
+		ids = _.filter ids, (id)-> id?
+		fileData = ids.join("\n")
+		fs.writeFile all_projects_path, fileData, ->
+			callback(err, ids)
+
+getProjectIds = (callback)->
+	exists = fs.existsSync all_projects_path
+	if exists 
+		loadProjectIds callback
+	else
+		getAndWriteProjectids callback
+
+markProjectAsProcessed = (project_id, callback)->
+	fs.appendFile finished_projects_path, "#{project_id}\n", callback
+
+getAllDocs = (project_id, callback = (error, docs) ->) ->
+	db.projects.findOne _id:ObjectId(project_id), (error, project) ->
+		return callback(error) if error?
+		if !project?
+			console.error("No such project: #{project_id}")
+			return callback("no such project #{project_id}")
+		findAllDocsInProject project, (error, docs) ->
+			return callback(error) if error?
+			return callback null, docs
+
+findAllDocsInProject = (project, callback = (error, docs) ->) ->
+	callback null, _findAllDocsInFolder project.rootFolder[0]
+
+_findDocInFolder = (folder = {}, doc_id, currentPath) ->
+	for doc, i in folder.docs or []
+		if doc?._id? and doc._id.toString() == doc_id.toString()
+			return {
+				doc: doc
+				mongoPath: "#{currentPath}.docs.#{i}"
+			}
+
+	for childFolder, i in folder.folders or []
+		result = _findDocInFolder childFolder, doc_id, "#{currentPath}.folders.#{i}"
+		return result if result?
+
+	return null
+
+_findAllDocsInFolder = (folder = {}) ->
+	docs = folder.docs or []
+	for childFolder in folder.folders or []
+		docs = docs.concat _findAllDocsInFolder childFolder
+	return docs
+
+insertDocIntoDocCollection = (project_id, doc_id, lines, oldRev, callback)->
+	update = {}
+	update["_id"] = ObjectId(doc_id)
+	update["lines"] = lines
+	update["project_id"] = ObjectId(project_id)
+	update["rev"] = oldRev
+	db.docs.insert _id: ObjectId(doc_id), callback
+
+saveDocsIntoMongo = (project_id, docs, callback)->
+	jobs = _.map docs, (doc)->
+		(cb)->
+			insertDocIntoDocCollection project_id, doc._id, project_id.lines, doc.rev, cb
+	async.series jobs, callback
+
+
+processNext = (project_id, callback)->
+	checkIfFileHasBeenProccessed project_id, (err, hasBeenProcessed)->
+		if hasBeenProcessed
+			console.log "#{project_id} already procssed, skipping"
+			return callback()
+		getAllDocs project_id, (err, docs)->
+			if err?
+				console.error err, project_id, "could not get all docs"
+				return callback()
+			saveDocsIntoMongo project_id, docs, ->
+				if err?
+					console.error err, project_id, "could not save docs into mongo"
+					return callback()
+				markProjectAsProcessed project_id, ->
+					callback()
+
+getProjectIds (err, ids)->
+	printProgress()
+	jobs = _.map ids, (id)->
+		return (cb)->
+			processNext(id, cb)
+	async.series jobs, (err)->
+		if err?
+			console.error err, "at end of jobs"
+		else
+			console.log "finished"
+		process.exit()
+	
+exports.up = (next)->
+
+	next()
+
+
+exports.down = (next)->
+	next()

package.json

@@ -1,11 +1,14 @@
 {
   "name": "sharelatex",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "An online collaborative LaTeX editor",
   "dependencies": {
     "async": "^0.9.0",
+    "lodash": "^3.0.0",
+    "mongojs": "^0.18.1",
     "rimraf": "~2.2.6",
-    "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git"
+    "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git",
+    "underscore": "^1.7.0"
   },
   "devDependencies": {
     "grunt": "~0.4.2",
@@ -14,6 +17,7 @@
     "grunt-execute": "~0.1.5",
     "grunt-available-tasks": "~0.4.1",
     "grunt-concurrent": "~0.4.3",
+    "grunt-contrib-coffee": "~0.10.1",
     "semver": "~2.2.1",
     "knox": "~0.8.9"
   }

package/upstart/sharelatex-chat

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-clsi

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-docstore

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-document-updater

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-filestore

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-spelling

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-tags

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-template

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-track-changes

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script

package/upstart/sharelatex-web

@@ -23,6 +23,6 @@ script
     LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
 
     echo $$ > /var/run/sharelatex-$SERVICE.pid
-    chdir /var/www/sharelatex/$SERVICE
+    cd /var/www/sharelatex/$SERVICE
     exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
 end script