alois/Verso
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into release

Browse Source
This commit is contained in:
James Allen
2015-02-26 11:51:15 +00:00
parent 5813e23fa8 4d01a54391
commit 7d2e86f07d
9 changed files with 131 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -12,6 +12,7 @@ tags
chat
spelling
real-time
migrations/*.js
data
tmp
CHANGELOG.md
+11
View File
@@ -0,0 +1,11 @@
v0.1.2
------
* Re-brand open-source ShareLaTeX code as 'ShareLaTeX Community Edition'.
* The Dropbox and template code has been extracted out into a separate module and removed from the ShareLaTeX Community Edition. There should be no broken features due to lack of open source components now.
* Websockets and real-time data now go to a separate light-weight [real-time](https://github.com/sharelatex/real-time) service.
* Updated PDF viewer that loads page-by-page for much quicker loading times on large documents.
* Links are clickable in chat messages.
* Mathjax libraries are now served locally.
* Optimisation of Angular digest loop in editor should reduce CPU usage in certain cases.
* Numerous small bug fixes.
Gruntfile.coffee
+7 -2
View File
@@ -7,6 +7,8 @@ semver = require "semver"
knox = require "knox"
crypto = require "crypto"
async = require "async"
settings = require("settings-sharelatex")
SERVICES = [{
name: "web"
@@ -56,6 +58,7 @@ module.exports = (grunt) ->
grunt.loadNpmTasks 'grunt-available-tasks'
grunt.loadNpmTasks 'grunt-concurrent'
grunt.loadNpmTasks "grunt-contrib-coffee"
grunt.loadNpmTasks "grunt-shell"
execute = {}
@@ -83,7 +86,9 @@ module.exports = (grunt) ->
options:
bare:true
shell:
migrate:
command: "./node_modules/east/bin/east migrate --adapter east-mongo --url #{settings.mongo.url}"
availabletasks:
tasks:
@@ -162,7 +167,7 @@ module.exports = (grunt) ->
Helpers.buildUpstartScripts()
grunt.registerTask 'migrate', 'run migrations', ['coffee:migrate']
#grunt.registerTask 'migrate', "compile migrations and run them", ['coffee:migrate', 'shell:migrate']
Helpers =
README.md
+5
View File
@@ -15,6 +15,11 @@ We have detailed installation instructions in our wiki:
**If you have any problems, have a look at our page of [Frequent Problems and Questions](https://github.com/sharelatex/sharelatex/wiki/FAQ).**
Upgrading
---------
If you are upgrading from a previous version of ShareLaTeX, please see the [Release Notes section on the Wiki] (https://github.com/sharelatex/sharelatex/wiki/Home) for all of the versions between your current version and the version you are upgrading to.
Dependencies
------------
config/settings.development.coffee.example
+8
View File
@@ -93,6 +93,14 @@ module.exports =
# when emails are sent out and in generated links:
siteUrl : 'http://localhost:3000'
# The websocket layer of ShareLaTeX runs as separate service.
# When running locally or in development, you can point the client to this
# service directly on port 3026. If you are running behind a reverse proxy (Nginx, etc)
# then websocketsUrl should be the same as siteUrl, with your reverse
# proxy responible for sending websocket traffic to the websocket service
# rather than connecting directly.
websocketsUrl: 'http://localhost:3026'
# If provided, a sessionSecret is used to sign cookies so that they cannot be
# spoofed. This is recommended.
security:
migrations/1422460849371-doc-lines.coffee → migrations/1_doc_lines.coffee
+66 -38
View File
@@ -1,31 +1,32 @@
Settings = require "settings-sharelatex"
fs = require("fs")
mongojs = require("mongojs")
ObjectId = mongojs.ObjectId
db = mongojs('sharelatex', ['projects', 'docs'])
console.log Settings.mongo.url
db = mongojs(Settings.mongo.url, ['projects', 'docs'])
_ = require("lodash")
async = require("async")
exec = require("child_process").exec
finished_projects_path = "/tmp/finished-projects"
all_projects_path = "/tmp/all-projects"
project_too_large_path = "/tmp/large_projects"
printProgress = ->
exec "wc #{finished_projects_path}", (error, results) ->
#console.log results
setTimeout printProgress, 1000 * 30
checkIfFileHasBeenProccessed = (project_id, callback)->
exec "grep #{project_id} #{finished_projects_path}", (error, results) ->
hasBeenProcessed = _.include(results, project_id)
#console.log hasBeenProcessed, project_id
callback(null, hasBeenProcessed)
callback(error, hasBeenProcessed)
loadProjectIds = (callback)->
console.log "loading project ids from #{all_projects_path}"
fs.readFile all_projects_path, "utf-8", (err, data)->
console.log data.length
ids = data.split("\n")
console.log ids.length
console.log "loaded #{ids.length} project ids from #{all_projects_path}"
callback err, ids
getAndWriteProjectids = (callback)->
@@ -38,9 +39,14 @@ getAndWriteProjectids = (callback)->
fs.writeFile all_projects_path, fileData, ->
callback(err, ids)
markProjectAsToLargeAndFinished = (project_id, callback)->
console.log "#{project_id} too large"
markProjectAsProcessed project_id, (err)->
fs.appendFile project_too_large_path, "#{project_id}\n", callback
getProjectIds = (callback)->
exists = fs.existsSync all_projects_path
if exists
if exists
loadProjectIds callback
else
getAndWriteProjectids callback
@@ -52,8 +58,11 @@ getAllDocs = (project_id, callback = (error, docs) ->) ->
db.projects.findOne _id:ObjectId(project_id), (error, project) ->
return callback(error) if error?
if !project?
console.error("No such project: #{project_id}")
return callback("no such project #{project_id}")
console.log "no such project #{project_id}"
return callback()
size = require("../node_modules/mongojs/node_modules/mongodb/node_modules/bson/").BSONPure.BSON.calculateObjectSize(project)
if size > 12000000 #12mb
return markProjectAsToLargeAndFinished project_id, callback
findAllDocsInProject project, (error, docs) ->
return callback(error) if error?
return callback null, docs
@@ -82,17 +91,33 @@ _findAllDocsInFolder = (folder = {}) ->
return docs
insertDocIntoDocCollection = (project_id, doc_id, lines, oldRev, callback)->
if !project_id?
return callback("no project id")
if !doc_id?
return callback()
if !lines?
lines = [""]
update = {}
update["_id"] = ObjectId(doc_id)
update["_id"] = ObjectId(doc_id.toString())
update["lines"] = lines
update["project_id"] = ObjectId(project_id)
update["rev"] = oldRev
db.docs.insert _id: ObjectId(doc_id), callback
update["rev"] = oldRev || 0
db.docs.insert update, callback
saveDocsIntoMongo = (project_id, docs, callback)->
jobs = _.map docs, (doc)->
(cb)->
insertDocIntoDocCollection project_id, doc._id, project_id.lines, doc.rev, cb
if !doc?
console.error "null doc in project #{project_id}" #just skip it, not a big deal
return cb()
insertDocIntoDocCollection project_id, doc._id, doc.lines, doc.rev, (err)->
if err?.code == 11000 #duplicate key, doc already in there so its not a problem.
err = undefined
if err?
console.log "error inserting doc into doc collection", err
cb(err)
async.series jobs, callback
@@ -101,33 +126,36 @@ processNext = (project_id, callback)->
if hasBeenProcessed
console.log "#{project_id} already procssed, skipping"
return callback()
console.log "#{project_id} processing"
getAllDocs project_id, (err, docs)->
if err?
console.error err, project_id, "could not get all docs"
return callback()
saveDocsIntoMongo project_id, docs, ->
if err?
console.error err, project_id, "could not save docs into mongo"
return callback()
markProjectAsProcessed project_id, ->
callback()
return callback(err)
else
saveDocsIntoMongo project_id, docs, (err)->
if err?
console.error err, project_id, "could not save docs into mongo"
return callback(err)
markProjectAsProcessed project_id, (err)->
setTimeout(
-> callback(err)
,500)
getProjectIds (err, ids)->
printProgress()
jobs = _.map ids, (id)->
return (cb)->
processNext(id, cb)
async.series jobs, (err)->
if err?
console.error err, "at end of jobs"
else
console.log "finished"
process.exit()
exports.up = (next)->
next()
exports.down = (next)->
exports.migrate = (client, done = ->)->
getProjectIds (err, ids)->
printProgress()
jobs = _.map ids, (id)->
return (cb)->
processNext(id, cb)
async.series jobs, (err)->
if err?
console.error err, "at end of jobs"
else
console.log "finished"
done(err)
exports.rollback = (next)->
next()
migrations/about_migrations.md
+2
View File
@@ -0,0 +1,2 @@
* if migration is stopped mid way it will start at the beginging next time
* to see the run migrations do db.getCollection('_migrations').find() you can't do db._migrations.find()
package.json
+3
View File
@@ -4,6 +4,9 @@
"description": "An online collaborative LaTeX editor",
"dependencies": {
"async": "^0.9.0",
"east": "^0.2.3",
"east-mongo": "^0.1.2",
"grunt-shell": "^1.1.1",
"lodash": "^3.0.0",
"mongojs": "^0.18.1",
"rimraf": "~2.2.6",
package/upstart/sharelatex-real-time
+28
View File
@@ -0,0 +1,28 @@
description "sharelatex-web"
author "ShareLaTeX <team@sharelatex.com>"
start on (local-filesystems and net-device-up IFACE!=lo)
stop on shutdown
respawn
limit nofile 8192 8192
pre-start script
mkdir -p /var/log/sharelatex
end script
script
SERVICE=real-time
USER=sharelatex
GROUP=sharelatex
# You may need to replace this with an absolute
# path to Node.js if it's not in your system PATH.
NODE=node
SHARELATEX_CONFIG=/etc/sharelatex/settings.coffee
LATEX_PATH=/usr/local/texlive/2014/bin/x86_64-linux
echo $$ > /var/run/sharelatex-$SERVICE.pid
cd /var/www/sharelatex/$SERVICE
exec sudo -u $USER -g $GROUP env SHARELATEX_CONFIG=$SHARELATEX_CONFIG NODE_ENV=production PATH=$PATH:$LATEX_PATH $NODE app.js >> /var/log/sharelatex/$SERVICE.log 2>&1
end script