alois/Verso
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #44 from sharelatex/ho-docker-no-keychange

Browse Source 
docker build script minimal
This commit is contained in:
Henry Oswald
2019-02-12 13:45:49 +00:00
committed by GitHub
parent 78d04adfec 0a0fc91f28
commit fa40e2c95f
15 changed files with 1847 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/document-updater/.dockerignore
+8 -52
View File
@@ -1,53 +1,9 @@
compileFolder
Compiled source #
###################
*.com
*.class
*.dll
*.exe
*.o
*.so
# Packages #
############
# it's better to unpack these files and commit the raw source
# git has its own built in compression methods
*.7z
*.dmg
*.gz
*.iso
*.jar
*.rar
*.tar
*.zip
# Logs and databases #
######################
*.log
*.sql
*.sqlite
# OS generated files #
######################
.DS_Store?
ehthumbs.db
Icon?
Thumbs.db
/node_modules/*
node_modules/*
gitrev
.git
.gitignore
.npm
.nvmrc
nodemon.json
app.js
app/js/*
test/unit/js/*
test/acceptance/js/*
forever/
**.swp
# Redis cluster
**/appendonly.aof
**/dump.rdb
**/nodes.conf
**/js/*
services/document-updater/.gitignore
+1
View File
@@ -39,6 +39,7 @@ Thumbs.db
app.js
app/js/*
**/*.map
test/unit/js/*
test/acceptance/js/*
services/document-updater/Dockerfile
+22
View File
@@ -0,0 +1,22 @@
FROM node:6.9.5 as app
WORKDIR /app
#wildcard as some files may not be in all repos
COPY package*.json npm-shrink*.json /app/
RUN npm install --quiet
COPY . /app
RUN npm run compile:all
FROM node:6.9.5
COPY --from=app /app /app
WORKDIR /app
USER node
CMD ["node", "--expose-gc", "app.js"]
services/document-updater/Jenkinsfile
Vendored
+15 -27
View File
@@ -17,13 +17,6 @@ pipeline {
stages {
stage('Install') {
agent {
docker {
image 'node:6.9.5'
args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp"
reuseNode true
}
}
steps {
withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
sh "curl $GIT_API_URL \
@@ -34,24 +27,12 @@ pipeline {
\"context\": \"ci/jenkins\" }' \
-u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD"
}
// we need to disable logallrefupdates, else git clones
// during the npm install will require git to lookup the
// user id which does not exist in the container's
// /etc/passwd file, causing the clone to fail.
sh 'git config --global core.logallrefupdates false'
sh 'rm -rf node_modules'
sh 'npm install && npm rebuild'
}
}
stage('Compile') {
agent {
docker {
image 'node:6.9.5'
reuseNode true
}
}
stage('Build') {
steps {
sh 'npm run compile:all'
sh 'make build'
}
}
@@ -67,20 +48,27 @@ pipeline {
}
}
stage('Package and publish build') {
stage('Package and docker push') {
steps {
sh 'echo ${BUILD_NUMBER} > build_number.txt'
sh 'touch build.tar.gz' // Avoid tar warning about files changing during read
sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .'
withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz")
sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make tar'
withCredentials([file(credentialsId: 'gcr.io_overleaf-ops', variable: 'DOCKER_REPO_KEY_PATH')]) {
sh 'docker login -u _json_key --password-stdin https://gcr.io/overleaf-ops < ${DOCKER_REPO_KEY_PATH}'
}
sh 'DOCKER_REPO=gcr.io/overleaf-ops make publish'
sh 'docker logout https://gcr.io/overleaf-ops'
}
}
stage('Publish build number') {
stage('Publish to s3') {
steps {
sh 'echo ${BRANCH_NAME}-${BUILD_NUMBER} > build_number.txt'
withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz")
}
withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
// The deployment process uses this file to figure out the latest build
s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest")
services/document-updater/Makefile
+14 -2
View File
@@ -1,7 +1,7 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
# Version: 1.1.9
# Version: 1.1.12
BUILD_NUMBER ?= local
BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
@@ -13,8 +13,9 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
MOCHA_GREP=${MOCHA_GREP} \
docker-compose ${DOCKER_COMPOSE_FLAGS}
clean:
docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
rm -f app.js
rm -rf app/js
rm -rf test/unit/js
@@ -33,5 +34,16 @@ test_clean:
test_acceptance_pre_run:
@[ ! -f test/acceptance/scripts/pre-run ] && echo "document-updater has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/scripts/pre-run
build:
docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
--tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
.
tar:
$(DOCKER_COMPOSE) up tar
publish:
docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
.PHONY: clean test test_unit test_acceptance test_clean build publish
services/document-updater/app.coffee
+7 -4
View File
@@ -1,8 +1,11 @@
Metrics = require "metrics-sharelatex"
Metrics.initialize("doc-updater")
express = require('express')
http = require("http")
Settings = require('settings-sharelatex')
logger = require('logger-sharelatex')
logger.initialize("documentupdater")
logger.initialize("document-updater")
logger.logger.serializers.docs = require("./app/js/LoggerSerializers").docs
logger.logger.serializers.files = require("./app/js/LoggerSerializers").files
if Settings.sentry?.dsn?
@@ -14,8 +17,7 @@ Errors = require "./app/js/Errors"
HttpController = require "./app/js/HttpController"
Path = require "path"
Metrics = require "metrics-sharelatex"
Metrics.initialize("doc-updater")
Metrics.mongodb.monitor(Path.resolve(__dirname + "/node_modules/mongojs/node_modules/mongodb"), logger)
Metrics.event_loop.monitor(logger, 100)
@@ -24,6 +26,7 @@ app.configure ->
app.use(Metrics.http.monitor(logger));
app.use express.bodyParser()
app.use app.router
Metrics.injectMetricsRoute(app)
DispatchManager.createAndStartDispatchers(Settings.dispatcherCount || 10)
@@ -76,7 +79,7 @@ app.get "/health_check/redis", (req, res, next) ->
res.send 500
else
res.send 200
docUpdaterRedisClient = require("redis-sharelatex").createClient(Settings.redis.documentupdater)
app.get "/health_check/redis_cluster", (req, res, next) ->
docUpdaterRedisClient.healthCheck (error) ->
services/document-updater/app/coffee/DispatchManager.coffee
+1
View File
@@ -16,6 +16,7 @@ module.exports = DispatchManager =
_waitForUpdateThenDispatchWorker: (callback = (error) ->) ->
timer = new Metrics.Timer "worker.waiting"
worker.client.blpop "pending-updates-list", 0, (error, result) ->
logger.log("getting pending-updates-list", error, result)
timer.done()
return callback(error) if error?
return callback() if !result?
services/document-updater/buildscript.txt
+8
View File
@@ -0,0 +1,8 @@
document-updater
--language=coffeescript
--node-version=6.9.5
--acceptance-creds=None
--dependencies=mongo,redis
--docker-repos=gcr.io/overleaf-ops
--build-target=docker
--script-version=1.1.12
services/document-updater/config/settings.defaults.coffee
+56 -67
View File
@@ -10,7 +10,7 @@ module.exports =
apis:
web:
url: "http://#{process.env["WEB_HOST"] or "localhost"}:#{process.env['WEB_PORT'] or 3000}"
url: "http://#{process.env['WEB_API_HOST'] or process.env['WEB_HOST'] or "localhost"}:#{process.env['WEB_API_PORT'] or process.env['WEB_PORT'] or 3000}"
user: "sharelatex"
pass: "password"
trackchanges:
@@ -20,82 +20,71 @@ module.exports =
url: "http://#{process.env["PROJECT_HISTORY_HOST"] or "localhost"}:3054"
redis:
realtime:
port: "6379"
host: process.env["REDIS_HOST"] or "localhost"
password:""
key_schema:
realtime:
port: process.env["REAL_TIME_REDIS_PORT"] or process.env["REDIS_PORT"] or "6379"
host: process.env["REAL_TIME_REDIS_HOST"] or process.env["REDIS_HOST"] or "localhost"
password: process.env["REAL_TIME_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or ""
redisOptions:
keepAlive: 100
key_schema:
pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}"
# cluster: [{
# port: "7000"
# host: "localhost"
# }]
# key_schema:
# pendingUpdates: ({doc_id}) -> "PendingUpdates:{#{doc_id}}"
documentupdater:
port: "6379"
host: process.env["REDIS_HOST"] or "localhost"
password: ""
key_schema:
blockingKey: ({doc_id}) -> "Blocking:#{doc_id}"
docLines: ({doc_id}) -> "doclines:#{doc_id}"
docOps: ({doc_id}) -> "DocOps:#{doc_id}"
docVersion: ({doc_id}) -> "DocVersion:#{doc_id}"
docHash: ({doc_id}) -> "DocHash:#{doc_id}"
projectKey: ({doc_id}) -> "ProjectId:#{doc_id}"
docsInProject: ({project_id}) -> "DocsIn:#{project_id}"
ranges: ({doc_id}) -> "Ranges:#{doc_id}"
pathname: ({doc_id}) -> "Pathname:#{doc_id}"
projectHistoryId: ({doc_id}) -> "ProjectHistoryId:#{doc_id}"
projectState: ({project_id}) -> "ProjectState:#{project_id}"
unflushedTime: ({doc_id}) -> "UnflushedTime:#{doc_id}"
# cluster: [{
# port: "7000"
# host: "localhost"
# }]
# key_schema:
# blockingKey: ({doc_id}) -> "Blocking:{#{doc_id}}"
# docLines: ({doc_id}) -> "doclines:{#{doc_id}}"
# docOps: ({doc_id}) -> "DocOps:{#{doc_id}}"
# docVersion: ({doc_id}) -> "DocVersion:{#{doc_id}}"
# docHash: ({doc_id}) -> "DocHash:{#{doc_id}}"
# projectKey: ({doc_id}) -> "ProjectId:{#{doc_id}}"
# docsInProject: ({project_id}) -> "DocsIn:{#{project_id}}"
# ranges: ({doc_id}) -> "Ranges:{#{doc_id}}"
# projectState: ({project_id}) -> "ProjectState:{#{project_id}}"
history:
port: "6379"
host: process.env["REDIS_HOST"] or "localhost"
password:""
port: process.env["HISTORY_REDIS_PORT"] or process.env["REDIS_PORT"] or "6379"
host: process.env["HISTORY_REDIS_HOST"] or process.env["REDIS_HOST"] or "localhost"
password: process.env["HISTORY_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or ""
key_schema:
uncompressedHistoryOps: ({doc_id}) -> "UncompressedHistoryOps:#{doc_id}"
docsWithHistoryOps: ({project_id}) -> "DocsWithHistoryOps:#{project_id}"
uncompressedHistoryOps: ({doc_id}) -> "UncompressedHistoryOps:{#{doc_id}}"
docsWithHistoryOps: ({project_id}) -> "DocsWithHistoryOps:{#{project_id}}"
redisOptions:
keepAlive: 100
project_history:
port: process.env["HISTORY_REDIS_PORT"] or process.env["REDIS_PORT"] or "6379"
host: process.env["HISTORY_REDIS_HOST"] or process.env["REDIS_HOST"] or "localhost"
password: process.env["HISTORY_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or ""
key_schema:
projectHistoryOps: ({project_id}) -> "ProjectHistory:Ops:#{project_id}"
projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:#{project_id}"
# cluster: [{
# port: "7000"
# host: "localhost"
# }]
# key_schema:
# uncompressedHistoryOps: ({doc_id}) -> "UncompressedHistoryOps:{#{doc_id}}"
# docsWithHistoryOps: ({project_id}) -> "DocsWithHistoryOps:{#{project_id}}"
projectHistoryOps: ({project_id}) -> "ProjectHistory:Ops:{#{project_id}}"
projectHistoryFirstOpTimestamp: ({project_id}) -> "ProjectHistory:FirstOpTimestamp:{#{project_id}}"
redisOptions:
keepAlive: 100
lock:
port: "6379"
host: process.env["REDIS_HOST"] or "localhost"
password:""
port: process.env["LOCK_REDIS_PORT"] or process.env["REDIS_PORT"] or "6379"
host: process.env["LOCK_REDIS_HOST"] or process.env["REDIS_HOST"] or "localhost"
password: process.env["LOCK_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or ""
key_schema:
blockingKey: ({doc_id}) -> "Blocking:#{doc_id}"
# cluster: [{
# port: "7000"
# host: "localhost"
# }]
# key_schema:
# blockingKey: ({doc_id}) -> "Blocking:{#{doc_id}}"
blockingKey: ({doc_id}) -> "Blocking:{#{doc_id}}"
redisOptions:
keepAlive: 100
documentupdater:
port: process.env["DOC_UPDATER_REDIS_PORT"] or process.env["REDIS_PORT"] or "6379"
host: process.env["DOC_UPDATER_REDIS_HOST"] or process.env["REDIS_HOST"] or "localhost"
password: process.env["DOC_UPDATER_REDIS_PASSWORD"] or process.env["REDIS_PASSWORD"] or ""
redisOptions:
keepAlive: 100
key_schema:
blockingKey: ({doc_id}) -> "Blocking:{#{doc_id}}"
docLines: ({doc_id}) -> "doclines:{#{doc_id}}"
docOps: ({doc_id}) -> "DocOps:{#{doc_id}}"
docVersion: ({doc_id}) -> "DocVersion:{#{doc_id}}"
docHash: ({doc_id}) -> "DocHash:{#{doc_id}}"
projectKey: ({doc_id}) -> "ProjectId:{#{doc_id}}"
docsInProject: ({project_id}) -> "DocsIn:{#{project_id}}"
ranges: ({doc_id}) -> "Ranges:{#{doc_id}}"
unflushedTime: ({doc_id}) -> "UnflushedTime:{#{doc_id}}"
pathname: ({doc_id}) -> "Pathname:{#{doc_id}}"
projectHistoryId: ({doc_id}) -> "ProjectHistoryId:{#{doc_id}}"
projectState: ({project_id}) -> "ProjectState:{#{project_id}}"
pendingUpdates: ({doc_id}) -> "PendingUpdates:{#{doc_id}}"
redisOptions:
keepAlive: 100
max_doc_length: 2 * 1024 * 1024 # 2mb
dispatcherCount: process.env["DISPATCHER_COUNT"]
mongo:
url: "mongodb://#{process.env["MONGO_HOST"] or "localhost"}/sharelatex"
services/document-updater/docker-compose.ci.yml
+15 -9
View File
@@ -1,24 +1,20 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
# Version: 1.1.9
# Version: 1.1.12
version: "2"
services:
test_unit:
image: node:6.9.5
volumes:
- .:/app
working_dir: /app
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
user: node
command: npm run test:unit:_run
test_acceptance:
image: node:6.9.5
volumes:
- .:/app
working_dir: /app
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
ELASTIC_SEARCH_DSN: es:9200
REDIS_HOST: redis
@@ -31,6 +27,16 @@ services:
user: node
command: npm run test:acceptance:_run
tar:
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
volumes:
- ./:/tmp/build/
command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
user: root
redis:
image: redis
services/document-updater/docker-compose.yml
+13 -3
View File
@@ -1,13 +1,13 @@
# This file was auto-generated, do not edit it directly.
# Instead run bin/update_build_scripts from
# https://github.com/sharelatex/sharelatex-dev-environment
# Version: 1.1.9
# Version: 1.1.12
version: "2"
services:
test_unit:
image: node:6.9.5
build: .
volumes:
- .:/app
working_dir: /app
@@ -17,7 +17,7 @@ services:
user: node
test_acceptance:
image: node:6.9.5
build: .
volumes:
- .:/app
working_dir: /app
@@ -33,6 +33,16 @@ services:
- redis
command: npm run test:acceptance
tar:
build: .
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
volumes:
- ./:/tmp/build/
command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
user: root
redis:
image: redis
services/document-updater/npm-shrinkwrap.json
Generated
+1675
View File
File diff suppressed because it is too large Load Diff
services/document-updater/package.json
+5 -5
View File
@@ -7,7 +7,7 @@
"url": "https://github.com/sharelatex/document-updater-sharelatex.git"
},
"scripts": {
"compile:app": "([ -e app/coffee ] && coffee $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')",
"compile:app": "([ -e app/coffee ] && coffee -m $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee -m $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')",
"start": "npm run compile:app && node $NODE_APP_OPTIONS app.js",
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js",
"test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
@@ -23,14 +23,14 @@
"async": "^2.5.0",
"coffee-script": "~1.7.0",
"express": "3.3.4",
"logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.6",
"logger-sharelatex": "^1.6.0",
"lynx": "0.0.11",
"metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.5.0",
"redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.4",
"metrics-sharelatex": "^2.1.1",
"redis-sharelatex": "^1.0.5",
"request": "2.25.0",
"requestretry": "^1.12.0",
"sandboxed-module": "~0.2.0",
"settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0",
"settings-sharelatex": "^1.1.0",
"sinon": "~1.5.2",
"underscore": "1.2.2"
},
services/document-updater/test/unit/coffee/DispatchManager/DispatchManagerTests.coffee
+5 -1
View File
@@ -4,8 +4,10 @@ should = chai.should()
modulePath = "../../../../app/js/DispatchManager.js"
SandboxedModule = require('sandboxed-module')
describe "DispatchManager", ->
beforeEach ->
@timeout(3000)
@DispatchManager = SandboxedModule.require modulePath, requires:
"./UpdateManager" : @UpdateManager = {}
"logger-sharelatex": @logger = { log: sinon.stub() }
@@ -14,6 +16,9 @@ describe "DispatchManager", ->
realtime: {}
"redis-sharelatex": @redis = {}
"./RateLimitManager": {}
"./Metrics":
Timer: ->
done: ->
@callback = sinon.stub()
@RateLimiter = { run: (task,cb) -> task(cb) } # run task without rate limit
@@ -22,7 +27,6 @@ describe "DispatchManager", ->
@client =
auth: sinon.stub()
@redis.createClient = sinon.stub().returns @client
@worker = @DispatchManager.createDispatcher(@RateLimiter)
it "should create a new redis client", ->
services/document-updater/test/unit/coffee/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.coffee
+2
View File
@@ -24,6 +24,8 @@ describe "ProjectHistoryRedisManager", ->
}
"redis-sharelatex":
createClient: () => @rclient
"logger-sharelatex":
log:->
globals:
JSON: @JSON = JSON