From c5e1584fcc9b94e40eb19d5fe929ca9bbd2f535f Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 2 May 2019 02:04:59 +0200 Subject: [PATCH 01/73] [FSPersistorManager] fix the stream opening for node10+ Attaching a `readable` listener causes the stream to hang otherwise. Signed-off-by: Jakob Ackermann --- services/filestore/app/coffee/FSPersistorManager.coffee | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/filestore/app/coffee/FSPersistorManager.coffee b/services/filestore/app/coffee/FSPersistorManager.coffee index 61e497a4fb..7d17f6d10a 100644 --- a/services/filestore/app/coffee/FSPersistorManager.coffee +++ b/services/filestore/app/coffee/FSPersistorManager.coffee @@ -48,10 +48,10 @@ module.exports = fs.open "#{location}/#{filteredName}", 'r', (err, fd) -> if err? logger.err err:err, location:location, filteredName:name, "Error reading from file" - if err.code == 'ENOENT' - return callback new Errors.NotFoundError(err.message), null - else - return callback err, null + if err.code == 'ENOENT' + return callback new Errors.NotFoundError(err.message), null + else + return callback err, null opts.fd = fd sourceStream = fs.createReadStream null, opts return callback null, sourceStream From ed97fcfcd00099758738bc9c8083e19dde9e94fd Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 2 May 2019 01:26:35 +0200 Subject: [PATCH 02/73] [logging] do not overwrite the logger name Signed-off-by: Jakob Ackermann --- services/filestore/app/coffee/FSPersistorManager.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/filestore/app/coffee/FSPersistorManager.coffee b/services/filestore/app/coffee/FSPersistorManager.coffee index 7d17f6d10a..38e30f284f 100644 --- a/services/filestore/app/coffee/FSPersistorManager.coffee +++ b/services/filestore/app/coffee/FSPersistorManager.coffee @@ -36,7 +36,7 @@ module.exports = if err? logger.err location:location, target:target, fsPath:fsPath, err:err, "something went wrong writing stream to disk" return callback err - @sendFile location, target, fsPath, (err) -> + @sendFile location, target, fsPath, (err) -> # delete the temporary file created above and return the original error LocalFileWriter.deleteFile fsPath, () -> callback(err) From c9106eff95e5e77e0abf29321e19e2d3eca764c5 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Fri, 13 Dec 2019 15:46:35 +0000 Subject: [PATCH 03/73] Upgrade Docker image to node 10 --- services/filestore/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/filestore/Dockerfile b/services/filestore/Dockerfile index 2845544ae6..5a18b41699 100644 --- a/services/filestore/Dockerfile +++ b/services/filestore/Dockerfile @@ -1,4 +1,4 @@ -FROM node:6.9.5 as app +FROM node:10.17.0-jessie as app WORKDIR /app @@ -9,10 +9,9 @@ RUN npm install --quiet COPY . /app - RUN npm run compile:all -FROM node:6.9.5 +FROM node:10.17.0-jessie COPY --from=app /app /app From 1798efd4be149e7bcc4490d3cb2cef7e8846d074 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 16 Dec 2019 10:15:52 +0000 Subject: [PATCH 04/73] Install eslint and prettier --- services/filestore/npm-shrinkwrap.json | 4655 ++++++++++++++++++++---- services/filestore/package.json | 14 + 2 files changed, 4000 insertions(+), 669 deletions(-) diff --git a/services/filestore/npm-shrinkwrap.json b/services/filestore/npm-shrinkwrap.json index 66670c6e69..413c41dca4 100644 --- a/services/filestore/npm-shrinkwrap.json +++ b/services/filestore/npm-shrinkwrap.json @@ -1,1581 +1,4248 @@ { "name": "filestore-sharelatex", "version": "0.1.4", + "lockfileVersion": 1, + "requires": true, "dependencies": { + "@babel/code-frame": { + "version": "7.5.5", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.5.5.tgz", + "integrity": "sha512-27d4lZoomVyo51VegxI20xZPuSHusqbQag/ztrBC7wegWoQ1nLREPVSKSW8byhTlzTKyNE4ifaTA6lCp7JjpFw==", + "dev": true, + "requires": { + "@babel/highlight": "^7.0.0" + } + }, + "@babel/generator": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.7.4.tgz", + "integrity": "sha512-m5qo2WgdOJeyYngKImbkyQrnUN1mPceaG5BV+G0E3gWsa4l/jCSryWJdM2x8OuGAOyh+3d5pVYfZWCiNFtynxg==", + "dev": true, + "requires": { + "@babel/types": "^7.7.4", + "jsesc": "^2.5.1", + "lodash": "^4.17.13", + "source-map": "^0.5.0" + }, + "dependencies": { + "lodash": { + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", + "dev": true + }, + "source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=", + "dev": true + } + } + }, + "@babel/helper-function-name": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.7.4.tgz", + "integrity": "sha512-AnkGIdiBhEuiwdoMnKm7jfPfqItZhgRaZfMg1XX3bS25INOnLPjPG1Ppnajh8eqgt5kPJnfqrRHqFqmjKDZLzQ==", + "dev": true, + "requires": { + "@babel/helper-get-function-arity": "^7.7.4", + "@babel/template": "^7.7.4", + "@babel/types": "^7.7.4" + } + }, + "@babel/helper-get-function-arity": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/@babel/helper-get-function-arity/-/helper-get-function-arity-7.7.4.tgz", + "integrity": "sha512-QTGKEdCkjgzgfJ3bAyRwF4yyT3pg+vDgan8DSivq1eS0gwi+KGKE5x8kRcbeFTb/673mkO5SN1IZfmCfA5o+EA==", + "dev": true, + "requires": { + "@babel/types": "^7.7.4" + } + }, + "@babel/helper-split-export-declaration": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.7.4.tgz", + "integrity": "sha512-guAg1SXFcVr04Guk9eq0S4/rWS++sbmyqosJzVs8+1fH5NI+ZcmkaSkc7dmtAFbHFva6yRJnjW3yAcGxjueDug==", + "dev": true, + "requires": { + "@babel/types": "^7.7.4" + } + }, + "@babel/highlight": { + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.5.0.tgz", + "integrity": "sha512-7dV4eu9gBxoM0dAnj/BCFDW9LFU0zvTrkq0ugM7pnHEgguOEeOz1so2ZghEdzviYzQEED0r4EAgpsBChKy1TRQ==", + "dev": true, + "requires": { + "chalk": "^2.0.0", + "esutils": "^2.0.2", + "js-tokens": "^4.0.0" + } + }, + "@babel/parser": { + "version": "7.7.5", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.7.5.tgz", + "integrity": "sha512-KNlOe9+/nk4i29g0VXgl8PEXIRms5xKLJeuZ6UptN0fHv+jDiriG+y94X6qAgWTR0h3KaoM1wK5G5h7MHFRSig==", + "dev": true + }, + "@babel/template": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.7.4.tgz", + "integrity": "sha512-qUzihgVPguAzXCK7WXw8pqs6cEwi54s3E+HrejlkuWO6ivMKx9hZl3Y2fSXp9i5HgyWmj7RKP+ulaYnKM4yYxw==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "@babel/parser": "^7.7.4", + "@babel/types": "^7.7.4" + } + }, + "@babel/traverse": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.7.4.tgz", + "integrity": "sha512-P1L58hQyupn8+ezVA2z5KBm4/Zr4lCC8dwKCMYzsa5jFMDMQAzaBNy9W5VjB+KAmBjb40U7a/H6ao+Xo+9saIw==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.5.5", + "@babel/generator": "^7.7.4", + "@babel/helper-function-name": "^7.7.4", + "@babel/helper-split-export-declaration": "^7.7.4", + "@babel/parser": "^7.7.4", + "@babel/types": "^7.7.4", + "debug": "^4.1.0", + "globals": "^11.1.0", + "lodash": "^4.17.13" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "dev": true + }, + "lodash": { + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", + "dev": true + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + } + } + }, + "@babel/types": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.7.4.tgz", + "integrity": "sha512-cz5Ji23KCi4T+YIE/BolWosrJuSmoZeN1EFnRtBwF+KKLi8GG/Z2c2hOJJeCXPk4mwk4QFvTmwIodJowXgttRA==", + "dev": true, + "requires": { + "esutils": "^2.0.2", + "lodash": "^4.17.13", + "to-fast-properties": "^2.0.0" + }, + "dependencies": { + "lodash": { + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", + "dev": true + } + } + }, "@google-cloud/common": { "version": "0.32.1", - "from": "@google-cloud/common@>=0.32.0 <0.33.0", - "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.32.1.tgz" + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.32.1.tgz", + "integrity": "sha1-ajLDQBcs6j22Z00ODjTnh0CgBz8=", + "requires": { + "@google-cloud/projectify": "^0.3.3", + "@google-cloud/promisify": "^0.4.0", + "@types/request": "^2.48.1", + "arrify": "^2.0.0", + "duplexify": "^3.6.0", + "ent": "^2.2.0", + "extend": "^3.0.2", + "google-auth-library": "^3.1.1", + "pify": "^4.0.1", + "retry-request": "^4.0.0", + "teeny-request": "^3.11.3" + } }, "@google-cloud/debug-agent": { "version": "3.2.0", - "from": "@google-cloud/debug-agent@>=3.0.0 <4.0.0", "resolved": "https://registry.npmjs.org/@google-cloud/debug-agent/-/debug-agent-3.2.0.tgz", + "integrity": "sha1-2qdjWhaYpWY31dxXzhED536uKdM=", + "requires": { + "@google-cloud/common": "^0.32.0", + "@sindresorhus/is": "^0.15.0", + "acorn": "^6.0.0", + "coffeescript": "^2.0.0", + "console-log-level": "^1.4.0", + "extend": "^3.0.1", + "findit2": "^2.2.3", + "gcp-metadata": "^1.0.0", + "lodash.pickby": "^4.6.0", + "p-limit": "^2.2.0", + "pify": "^4.0.1", + "semver": "^6.0.0", + "source-map": "^0.6.1", + "split": "^1.0.0" + }, "dependencies": { "coffeescript": { "version": "2.4.1", - "from": "coffeescript@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/coffeescript/-/coffeescript-2.4.1.tgz" + "resolved": "https://registry.npmjs.org/coffeescript/-/coffeescript-2.4.1.tgz", + "integrity": "sha1-gV/TN98KNNSedKmKbr6pw+eTD3A=" } } }, "@google-cloud/profiler": { "version": "0.2.3", - "from": "@google-cloud/profiler@>=0.2.3 <0.3.0", "resolved": "https://registry.npmjs.org/@google-cloud/profiler/-/profiler-0.2.3.tgz", + "integrity": "sha1-Fj3738Mwuug1X+RuHlvgZTV7H1w=", + "requires": { + "@google-cloud/common": "^0.26.0", + "@types/console-log-level": "^1.4.0", + "@types/semver": "^5.5.0", + "bindings": "^1.2.1", + "console-log-level": "^1.4.0", + "delay": "^4.0.1", + "extend": "^3.0.1", + "gcp-metadata": "^0.9.0", + "nan": "^2.11.1", + "parse-duration": "^0.1.1", + "pify": "^4.0.0", + "pretty-ms": "^4.0.0", + "protobufjs": "~6.8.6", + "semver": "^5.5.0", + "teeny-request": "^3.3.0" + }, "dependencies": { "@google-cloud/common": { "version": "0.26.2", - "from": "@google-cloud/common@>=0.26.0 <0.27.0", - "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.26.2.tgz" + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.26.2.tgz", + "integrity": "sha1-nFTiRxqEqgMelaJIJJduCA8lVkU=", + "requires": { + "@google-cloud/projectify": "^0.3.2", + "@google-cloud/promisify": "^0.3.0", + "@types/duplexify": "^3.5.0", + "@types/request": "^2.47.0", + "arrify": "^1.0.1", + "duplexify": "^3.6.0", + "ent": "^2.2.0", + "extend": "^3.0.1", + "google-auth-library": "^2.0.0", + "pify": "^4.0.0", + "retry-request": "^4.0.0", + "through2": "^3.0.0" + } }, "@google-cloud/promisify": { "version": "0.3.1", - "from": "@google-cloud/promisify@>=0.3.0 <0.4.0", - "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.3.1.tgz" + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.3.1.tgz", + "integrity": "sha1-9kHm2USo4KBe4MsQkd+mAIm+zbo=" }, "arrify": { "version": "1.0.1", - "from": "arrify@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz" + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", + "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=" }, "gcp-metadata": { "version": "0.9.3", - "from": "gcp-metadata@>=0.9.0 <0.10.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.9.3.tgz" + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.9.3.tgz", + "integrity": "sha1-H510lfdGChRSZIHynhFZbdVj3SY=", + "requires": { + "gaxios": "^1.0.2", + "json-bigint": "^0.3.0" + } }, "google-auth-library": { "version": "2.0.2", - "from": "google-auth-library@>=2.0.0 <3.0.0", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-2.0.2.tgz", + "integrity": "sha1-ejFdIDZ0Svavyth7IQ7mY4tA9Xs=", + "requires": { + "axios": "^0.18.0", + "gcp-metadata": "^0.7.0", + "gtoken": "^2.3.0", + "https-proxy-agent": "^2.2.1", + "jws": "^3.1.5", + "lru-cache": "^5.0.0", + "semver": "^5.5.0" + }, "dependencies": { "gcp-metadata": { "version": "0.7.0", - "from": "gcp-metadata@>=0.7.0 <0.8.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz" + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz", + "integrity": "sha1-bDXbtSvaMqQnu5yY9UI33dG1QG8=", + "requires": { + "axios": "^0.18.0", + "extend": "^3.0.1", + "retry-axios": "0.3.2" + } } } }, "nan": { "version": "2.14.0", - "from": "nan@>=2.11.1 <3.0.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz" + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz", + "integrity": "sha1-eBj3IgJ7JFmobwKV1DTR/CM2xSw=" }, "readable-stream": { "version": "3.4.0", - "from": "readable-stream@>=2.0.0 <3.0.0||>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.4.0.tgz" + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.4.0.tgz", + "integrity": "sha1-pRwmdUZY4KPCHb9ZFjvUW6b0R/w=", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } }, "semver": { "version": "5.7.0", - "from": "semver@>=5.5.0 <6.0.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz" + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", + "integrity": "sha1-eQp89v6lRZuslhELKbYEEtyP+Ws=" }, "string_decoder": { "version": "1.2.0", - "from": "string_decoder@>=1.1.1 <2.0.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.2.0.tgz" + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.2.0.tgz", + "integrity": "sha1-/obnOLGVRK/nBGkkOyoe6SQOro0=", + "requires": { + "safe-buffer": "~5.1.0" + } }, "through2": { "version": "3.0.1", - "from": "through2@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.1.tgz" + "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.1.tgz", + "integrity": "sha1-OSducTwzAu3544jdnIEt07glvVo=", + "requires": { + "readable-stream": "2 || 3" + } } } }, "@google-cloud/projectify": { "version": "0.3.3", - "from": "@google-cloud/projectify@>=0.3.3 <0.4.0", - "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-0.3.3.tgz" + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-0.3.3.tgz", + "integrity": "sha1-vekQPVCyCj6jM334xng6dm5w1B0=" }, "@google-cloud/promisify": { "version": "0.4.0", - "from": "@google-cloud/promisify@>=0.4.0 <0.5.0", - "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.4.0.tgz" + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.4.0.tgz", + "integrity": "sha1-T7/PTYW7ai5MzwWqY9KxDWyarZs=" }, "@google-cloud/trace-agent": { "version": "3.6.1", - "from": "@google-cloud/trace-agent@>=3.2.0 <4.0.0", "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-3.6.1.tgz", + "integrity": "sha1-W+dEE5TQ6ldY8o25IqUAT/PwO+w=", + "requires": { + "@google-cloud/common": "^0.32.1", + "builtin-modules": "^3.0.0", + "console-log-level": "^1.4.0", + "continuation-local-storage": "^3.2.1", + "extend": "^3.0.0", + "gcp-metadata": "^1.0.0", + "hex2dec": "^1.0.1", + "is": "^3.2.0", + "methods": "^1.1.1", + "require-in-the-middle": "^4.0.0", + "semver": "^6.0.0", + "shimmer": "^1.2.0", + "uuid": "^3.0.1" + }, "dependencies": { "uuid": { "version": "3.3.2", - "from": "uuid@>=3.0.1 <4.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz" + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha1-G0r0lV6zB3xQHCOHL8ZROBFYcTE=" } } }, "@protobufjs/aspromise": { "version": "1.1.2", - "from": "@protobufjs/aspromise@>=1.1.2 <2.0.0", - "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz" + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78=" }, "@protobufjs/base64": { "version": "1.1.2", - "from": "@protobufjs/base64@>=1.1.2 <2.0.0", - "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz" + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha1-TIVzDlm5ofHzSQR9vyQpYDS7JzU=" }, "@protobufjs/codegen": { "version": "2.0.4", - "from": "@protobufjs/codegen@>=2.0.4 <3.0.0", - "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz" + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha1-fvN/DQEPsCitGtWXIuUG2SYoFcs=" }, "@protobufjs/eventemitter": { "version": "1.1.0", - "from": "@protobufjs/eventemitter@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz" + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A=" }, "@protobufjs/fetch": { "version": "1.1.0", - "from": "@protobufjs/fetch@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz" + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=", + "requires": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } }, "@protobufjs/float": { "version": "1.0.2", - "from": "@protobufjs/float@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz" + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E=" }, "@protobufjs/inquire": { "version": "1.1.0", - "from": "@protobufjs/inquire@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz" + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik=" }, "@protobufjs/path": { "version": "1.1.2", - "from": "@protobufjs/path@>=1.1.2 <2.0.0", - "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz" + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0=" }, "@protobufjs/pool": { "version": "1.1.0", - "from": "@protobufjs/pool@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz" + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q=" }, "@protobufjs/utf8": { "version": "1.1.0", - "from": "@protobufjs/utf8@>=1.1.0 <2.0.0", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz" + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=" }, "@sindresorhus/is": { "version": "0.15.0", - "from": "@sindresorhus/is@>=0.15.0 <0.16.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.15.0.tgz" + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.15.0.tgz", + "integrity": "sha1-lpFbqgXmpqHRN7rfSYTT/AWCC7Y=" }, "@sinonjs/commons": { "version": "1.4.0", - "from": "@sinonjs/commons@>=1.2.0 <2.0.0", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.4.0.tgz", - "dev": true + "integrity": "sha1-ez7C2Wr0gdegMhJS57HJRyTsWng=", + "dev": true, + "requires": { + "type-detect": "4.0.8" + } }, "@sinonjs/formatio": { "version": "3.2.1", - "from": "@sinonjs/formatio@>=3.0.0 <4.0.0", "resolved": "https://registry.npmjs.org/@sinonjs/formatio/-/formatio-3.2.1.tgz", + "integrity": "sha1-UjEPL5vLxnvawYyUrUkBuV/eJn4=", "dev": true, + "requires": { + "@sinonjs/commons": "^1", + "@sinonjs/samsam": "^3.1.0" + }, "dependencies": { "@sinonjs/samsam": { "version": "3.3.2", - "from": "@sinonjs/samsam@>=3.1.0 <4.0.0", "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-3.3.2.tgz", - "dev": true + "integrity": "sha1-Y5QuPV6wt59t4775q/rRX7S2QBs=", + "dev": true, + "requires": { + "@sinonjs/commons": "^1.0.2", + "array-from": "^2.1.1", + "lodash": "^4.17.11" + } } } }, "@sinonjs/samsam": { "version": "2.1.3", - "from": "@sinonjs/samsam@>=2.1.2 <3.0.0", "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-2.1.3.tgz", + "integrity": "sha1-Ys8qm2JO3HlRNBNf43/Cro6ja+M=", "dev": true }, "@sinonjs/text-encoding": { "version": "0.7.1", - "from": "@sinonjs/text-encoding@>=0.7.1 <0.8.0", "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.1.tgz", + "integrity": "sha1-jaXGUwkVZT86Hzj9XxAdjD+AecU=", "dev": true }, "@types/caseless": { "version": "0.12.2", - "from": "@types/caseless@*", - "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz" + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", + "integrity": "sha1-9l09Y4ngHutFi9VNyPUrlalGO8g=" }, "@types/console-log-level": { "version": "1.4.0", - "from": "@types/console-log-level@>=1.4.0 <2.0.0", - "resolved": "https://registry.npmjs.org/@types/console-log-level/-/console-log-level-1.4.0.tgz" + "resolved": "https://registry.npmjs.org/@types/console-log-level/-/console-log-level-1.4.0.tgz", + "integrity": "sha1-7/ccQa689RyLpa2LBdfVQkviuPM=" }, "@types/duplexify": { "version": "3.6.0", - "from": "@types/duplexify@>=3.5.0 <4.0.0", - "resolved": "https://registry.npmjs.org/@types/duplexify/-/duplexify-3.6.0.tgz" + "resolved": "https://registry.npmjs.org/@types/duplexify/-/duplexify-3.6.0.tgz", + "integrity": "sha1-38grZL06IWj1vSZESvFlvwI33Ng=", + "requires": { + "@types/node": "*" + } + }, + "@types/eslint-visitor-keys": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz", + "integrity": "sha512-OCutwjDZ4aFS6PB1UZ988C4YgwlBHJd6wCeQqaLdmadZ/7e+w79+hbMUFC1QXDNCmdyoRfAFdm0RypzwR+Qpag==", + "dev": true }, "@types/form-data": { "version": "2.2.1", - "from": "@types/form-data@*", - "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-2.2.1.tgz" + "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-2.2.1.tgz", + "integrity": "sha1-7is7jqoRwJOCiZU2BrdFtzjFSx4=", + "requires": { + "@types/node": "*" + } + }, + "@types/json-schema": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.3.tgz", + "integrity": "sha512-Il2DtDVRGDcqjDtE+rF8iqg1CArehSK84HZJCT7AMITlyXRBpuPhqGLDQMowraqqu1coEaimg4ZOqggt6L6L+A==", + "dev": true }, "@types/long": { "version": "4.0.0", - "from": "@types/long@>=4.0.0 <5.0.0", - "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.0.tgz" + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.0.tgz", + "integrity": "sha1-cZVR0jUtMBrIuB23Mqy2vcKNve8=" }, "@types/node": { "version": "12.0.8", - "from": "@types/node@*", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.8.tgz" + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.8.tgz", + "integrity": "sha1-VRRmvhGyrcPz1HFWdY9hC9n2sdg=" }, "@types/request": { "version": "2.48.1", - "from": "@types/request@>=2.47.0 <3.0.0", - "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.1.tgz" + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.1.tgz", + "integrity": "sha1-5ALWkapmcPu/8ZV7FfEnAjCrQvo=", + "requires": { + "@types/caseless": "*", + "@types/form-data": "*", + "@types/node": "*", + "@types/tough-cookie": "*" + } }, "@types/semver": { "version": "5.5.0", - "from": "@types/semver@>=5.5.0 <6.0.0", - "resolved": "https://registry.npmjs.org/@types/semver/-/semver-5.5.0.tgz" + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-5.5.0.tgz", + "integrity": "sha1-FGwqKe59O65L8vyydGNuJkyBPEU=" }, "@types/tough-cookie": { "version": "2.3.5", - "from": "@types/tough-cookie@*", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.5.tgz" + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-2.3.5.tgz", + "integrity": "sha1-naRO11VxmZtlw3tgybK4jbVMWF0=" + }, + "@typescript-eslint/experimental-utils": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-1.13.0.tgz", + "integrity": "sha512-zmpS6SyqG4ZF64ffaJ6uah6tWWWgZ8m+c54XXgwFtUv0jNz8aJAVx8chMCvnk7yl6xwn8d+d96+tWp7fXzTuDg==", + "dev": true, + "requires": { + "@types/json-schema": "^7.0.3", + "@typescript-eslint/typescript-estree": "1.13.0", + "eslint-scope": "^4.0.0" + }, + "dependencies": { + "eslint-scope": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", + "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==", + "dev": true, + "requires": { + "esrecurse": "^4.1.0", + "estraverse": "^4.1.1" + } + } + } + }, + "@typescript-eslint/parser": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-1.13.0.tgz", + "integrity": "sha512-ITMBs52PCPgLb2nGPoeT4iU3HdQZHcPaZVw+7CsFagRJHUhyeTgorEwHXhFf3e7Evzi8oujKNpHc8TONth8AdQ==", + "dev": true, + "requires": { + "@types/eslint-visitor-keys": "^1.0.0", + "@typescript-eslint/experimental-utils": "1.13.0", + "@typescript-eslint/typescript-estree": "1.13.0", + "eslint-visitor-keys": "^1.0.0" + } + }, + "@typescript-eslint/typescript-estree": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-1.13.0.tgz", + "integrity": "sha512-b5rCmd2e6DCC6tCTN9GSUAuxdYwCM/k/2wdjHGrIRGPSJotWMCe/dGpi66u42bhuh8q3QBzqM4TMA1GUUCJvdw==", + "dev": true, + "requires": { + "lodash.unescape": "4.0.1", + "semver": "5.5.0" + } }, "abort-controller": { "version": "3.0.0", - "from": "abort-controller@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz" + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha1-6vVNU7YrrkE46AnKIlyEOabvs5I=", + "requires": { + "event-target-shim": "^5.0.0" + } }, "accept-encoding": { "version": "0.1.0", - "from": "accept-encoding@>=0.1.0 <0.2.0", - "resolved": "https://registry.npmjs.org/accept-encoding/-/accept-encoding-0.1.0.tgz" + "resolved": "https://registry.npmjs.org/accept-encoding/-/accept-encoding-0.1.0.tgz", + "integrity": "sha1-XdiLjfcfHcLlzGuVZezOHjmaMz4=" }, "accepts": { "version": "1.3.5", - "from": "accepts@>=1.3.5 <1.4.0", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz" + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz", + "integrity": "sha1-63d99gEXI6OxTopywIBcjoZ0a9I=", + "requires": { + "mime-types": "~2.1.18", + "negotiator": "0.6.1" + } }, "acorn": { "version": "6.1.1", - "from": "acorn@>=6.0.0 <7.0.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.1.1.tgz" + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.1.1.tgz", + "integrity": "sha1-fSWuBbuK0fm2mRCOEJTs14hK3B8=" + }, + "acorn-jsx": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.1.0.tgz", + "integrity": "sha512-tMUqwBWfLFbJbizRmEcWSLw6HnFzfdJs2sOJEOwwtVPMoH/0Ay+E703oZz78VSXZiiDcZrQ5XKjPIUQixhmgVw==", + "dev": true }, "agent-base": { "version": "4.3.0", - "from": "agent-base@>=4.1.0 <5.0.0", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz" + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz", + "integrity": "sha1-gWXwHENgCbzK0LHRIvBe13Dvxu4=", + "requires": { + "es6-promisify": "^5.0.0" + } }, "ajv": { "version": "6.10.0", - "from": "ajv@>=6.5.5 <7.0.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz" + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz", + "integrity": "sha1-kNDVRDnaWHzX6EO/twRfUL0ivfE=", + "requires": { + "fast-deep-equal": "^2.0.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "ansi-escapes": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.0.tgz", + "integrity": "sha512-EiYhwo0v255HUL6eDyuLrXEkTi7WwVCLAw+SeOQ7M7qdun1z1pum4DEm/nuqIVbPvi9RPPc9k9LbyBv6H0DwVg==", + "dev": true, + "requires": { + "type-fest": "^0.8.1" + } + }, + "ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true + }, + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "requires": { + "color-convert": "^1.9.0" + } + }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "requires": { + "sprintf-js": "~1.0.2" + } }, "array-flatten": { "version": "1.1.1", - "from": "array-flatten@1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz" + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" }, "array-from": { "version": "2.1.1", - "from": "array-from@>=2.1.1 <3.0.0", "resolved": "https://registry.npmjs.org/array-from/-/array-from-2.1.1.tgz", + "integrity": "sha1-z+nYwmYoudxa7MYqn12PHzUsEZU=", "dev": true }, + "array-includes": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.0.tgz", + "integrity": "sha512-ONOEQoKrvXPKk7Su92Co0YMqYO32FfqJTzkKU9u2UpIXyYZIzLSvpdg4AwvSw4mSUW0czu6inK+zby6Oj6gDjQ==", + "dev": true, + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.0" + } + }, + "array.prototype.flat": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.2.3.tgz", + "integrity": "sha512-gBlRZV0VSmfPIeWfuuy56XZMvbVfbEUnOXUvt3F/eUUUSyzlgLxhEX4YAEpxNAogRGehPSnfXyPtYyKAhkzQhQ==", + "dev": true, + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.1" + } + }, "arrify": { "version": "2.0.1", - "from": "arrify@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz" + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha1-yWVekzHgq81YjSp8rX6ZVvZnAfo=" }, "asn1": { "version": "0.2.4", - "from": "asn1@>=0.2.3 <0.3.0", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz" + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", + "integrity": "sha1-jSR136tVO7M+d7VOWeiAu4ziMTY=", + "requires": { + "safer-buffer": "~2.1.0" + } }, "assert-plus": { "version": "1.0.0", - "from": "assert-plus@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" }, "assertion-error": { "version": "1.1.0", - "from": "assertion-error@>=1.1.0 <2.0.0", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", + "integrity": "sha1-5gtrDo8wG9l+U3UhW9pAbIURjAs=", + "dev": true + }, + "astral-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-1.0.0.tgz", + "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", "dev": true }, "async": { "version": "0.2.10", - "from": "async@>=0.2.10 <0.3.0", - "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz" + "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz", + "integrity": "sha1-trvgsGdLnXGXCMo43owjfLUmw9E=" }, "async-listener": { "version": "0.6.10", - "from": "async-listener@>=0.6.0 <0.7.0", "resolved": "https://registry.npmjs.org/async-listener/-/async-listener-0.6.10.tgz", + "integrity": "sha1-p8l6vlcLpgLXgic8DeYKUePhfLw=", + "requires": { + "semver": "^5.3.0", + "shimmer": "^1.1.0" + }, "dependencies": { "semver": { "version": "5.7.0", - "from": "semver@>=5.3.0 <6.0.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz" + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", + "integrity": "sha1-eQp89v6lRZuslhELKbYEEtyP+Ws=" } } }, "asynckit": { "version": "0.4.0", - "from": "asynckit@>=0.4.0 <0.5.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz" + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" }, "aws-sdk": { "version": "2.315.0", - "from": "aws-sdk@>=2.1.39 <3.0.0", "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.315.0.tgz", + "integrity": "sha1-fzkxYq8DjL73IjdERKm8muG9u+k=", + "requires": { + "buffer": "4.9.1", + "events": "1.1.1", + "ieee754": "1.1.8", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.1.0", + "xml2js": "0.4.19" + }, "dependencies": { "uuid": { "version": "3.1.0", - "from": "uuid@3.1.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz" + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz", + "integrity": "sha1-PdPT55Crwk17DToDT/q6vijrvAQ=" } } }, "aws-sign": { "version": "0.2.1", - "from": "aws-sign@>=0.2.0 <0.3.0", - "resolved": "https://registry.npmjs.org/aws-sign/-/aws-sign-0.2.1.tgz" + "resolved": "https://registry.npmjs.org/aws-sign/-/aws-sign-0.2.1.tgz", + "integrity": "sha1-uWGyLwuqTxXsJBFA83dtbBQoVtA=" }, "aws-sign2": { "version": "0.7.0", - "from": "aws-sign2@>=0.7.0 <0.8.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz" + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" }, "aws4": { "version": "1.8.0", - "from": "aws4@>=1.8.0 <2.0.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz" + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", + "integrity": "sha1-8OAD2cqef1nHpQiUXXsu+aBKVC8=" }, "axios": { "version": "0.18.1", - "from": "axios@>=0.18.0 <0.19.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.1.tgz" + "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.1.tgz", + "integrity": "sha1-/z8N4ue10YDnV62YAA8Qgbh7zqM=", + "requires": { + "follow-redirects": "1.5.10", + "is-buffer": "^2.0.2" + } + }, + "babel-eslint": { + "version": "10.0.3", + "resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.0.3.tgz", + "integrity": "sha512-z3U7eMY6r/3f3/JB9mTsLjyxrv0Yb1zb8PCWCLpguxfCzBIZUwy23R1t/XKewP+8mEN2Ck8Dtr4q20z6ce6SoA==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "@babel/parser": "^7.0.0", + "@babel/traverse": "^7.0.0", + "@babel/types": "^7.0.0", + "eslint-visitor-keys": "^1.0.0", + "resolve": "^1.12.0" + }, + "dependencies": { + "resolve": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.13.1.tgz", + "integrity": "sha512-CxqObCX8K8YtAhOBRg+lrcdn+LK+WYOS8tSjqSFbjtrI5PnS63QPhZl4+yKfrU9tdsbMu9Anr/amegT87M9Z6w==", + "dev": true, + "requires": { + "path-parse": "^1.0.6" + } + } + } }, "balanced-match": { "version": "1.0.0", - "from": "balanced-match@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, "base64-js": { "version": "1.3.0", - "from": "base64-js@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz" + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz", + "integrity": "sha1-yrHmEY8FEJXli1KBrqjBzSK/wOM=" }, "bcrypt-pbkdf": { "version": "1.0.2", - "from": "bcrypt-pbkdf@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz" + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", + "requires": { + "tweetnacl": "^0.14.3" + } }, "best-encoding": { "version": "0.1.1", - "from": "best-encoding@>=0.1.1 <0.2.0", - "resolved": "https://registry.npmjs.org/best-encoding/-/best-encoding-0.1.1.tgz" + "resolved": "https://registry.npmjs.org/best-encoding/-/best-encoding-0.1.1.tgz", + "integrity": "sha1-GVIT2rysBFgYuAe3ox+Dn63cl04=", + "requires": { + "accept-encoding": "~0.1.0" + } }, "bignumber.js": { "version": "7.2.1", - "from": "bignumber.js@>=7.0.0 <8.0.0", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz" + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz", + "integrity": "sha1-gMBIdZ2CaACAfEv9Uh5Q7bulel8=" }, "bindings": { "version": "1.5.0", - "from": "bindings@>=1.2.1 <2.0.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz" + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha1-EDU8npRTNLwFEabZCzj7x8nFBN8=", + "requires": { + "file-uri-to-path": "1.0.0" + } }, "bintrees": { "version": "1.0.1", - "from": "bintrees@1.0.1", - "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz" + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz", + "integrity": "sha1-DmVcm5wkNeqraL9AJyJtK1WjRSQ=" }, "bl": { "version": "0.7.0", - "from": "bl@>=0.7.0 <0.8.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-0.7.0.tgz" + "resolved": "https://registry.npmjs.org/bl/-/bl-0.7.0.tgz", + "integrity": "sha1-P7BnBgKsKHjrdw3CA58YNr5irls=", + "requires": { + "readable-stream": "~1.0.2" + } }, "body-parser": { "version": "1.18.3", - "from": "body-parser@>=1.2.0 <2.0.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz" + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz", + "integrity": "sha1-WykhmP/dVTs6DyDe0FkrlWlVyLQ=", + "requires": { + "bytes": "3.0.0", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.2", + "http-errors": "~1.6.3", + "iconv-lite": "0.4.23", + "on-finished": "~2.3.0", + "qs": "6.5.2", + "raw-body": "2.3.3", + "type-is": "~1.6.16" + } + }, + "boolify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/boolify/-/boolify-1.0.1.tgz", + "integrity": "sha1-tcCeF8rNET0Rt7s+04TMASmU2Gs=", + "dev": true }, "boom": { "version": "0.3.8", - "from": "boom@>=0.3.0 <0.4.0", - "resolved": "https://registry.npmjs.org/boom/-/boom-0.3.8.tgz" + "resolved": "https://registry.npmjs.org/boom/-/boom-0.3.8.tgz", + "integrity": "sha1-yM2wQUNZEnQWKMBE7Mcy0dF8Ceo=", + "requires": { + "hoek": "0.7.x" + } }, "brace-expansion": { "version": "1.1.11", - "from": "brace-expansion@>=1.1.7 <2.0.0", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz" + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha1-PH/L9SnYcibz0vUrlm/1Jx60Qd0=", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } }, "browser-stdout": { "version": "1.3.1", - "from": "browser-stdout@1.3.1", - "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz" + "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", + "integrity": "sha1-uqVZ7hTO1zRSIputcyZGfGH6vWA=" }, "buffer": { "version": "4.9.1", - "from": "buffer@4.9.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz" + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } }, "buffer-equal-constant-time": { "version": "1.0.1", - "from": "buffer-equal-constant-time@1.0.1", - "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz" + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=" }, "builtin-modules": { "version": "3.1.0", - "from": "builtin-modules@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.1.0.tgz" + "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.1.0.tgz", + "integrity": "sha1-qtl8FRMet2tltQ7yCOdYTNdqdIQ=" }, "bunyan": { "version": "1.5.1", - "from": "bunyan@1.5.1", "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.5.1.tgz", - "dev": true + "integrity": "sha1-X259RMQ7lS9WsPQTCeOrEjkbTi0=", + "dev": true, + "requires": { + "dtrace-provider": "~0.6", + "mv": "~2", + "safe-json-stringify": "~1" + } }, "bytes": { "version": "3.0.0", - "from": "bytes@3.0.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz" + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", + "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" + }, + "callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true + }, + "camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true + }, + "camelcase-keys": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-6.1.1.tgz", + "integrity": "sha512-kEPCddRFChEzO0d6w61yh0WbBiSv9gBnfZWGfXRYPlGqIdIGef6HMR6pgqVSEWCYkrp8B0AtEpEXNY+Jx0xk1A==", + "dev": true, + "requires": { + "camelcase": "^5.3.1", + "map-obj": "^4.0.0", + "quick-lru": "^4.0.1" + } }, "caseless": { "version": "0.3.0", - "from": "caseless@>=0.3.0 <0.4.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.3.0.tgz" + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.3.0.tgz", + "integrity": "sha1-U06XkWOH07cGtk/eu6xGQ4RQk08=" }, "chai": { "version": "4.2.0", - "from": "chai@4.2.0", "resolved": "https://registry.npmjs.org/chai/-/chai-4.2.0.tgz", + "integrity": "sha1-dgqnLPION5XoSxKHfODoNzeqKeU=", + "dev": true, + "requires": { + "assertion-error": "^1.1.0", + "check-error": "^1.0.2", + "deep-eql": "^3.0.1", + "get-func-name": "^2.0.0", + "pathval": "^1.1.0", + "type-detect": "^4.0.5" + } + }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "chardet": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", + "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", "dev": true }, "check-error": { "version": "1.0.2", - "from": "check-error@>=1.0.2 <2.0.0", "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", + "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=", "dev": true }, + "cli-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", + "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", + "dev": true, + "requires": { + "restore-cursor": "^3.1.0" + } + }, + "cli-width": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.0.tgz", + "integrity": "sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk=", + "dev": true + }, + "cliui": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-5.0.0.tgz", + "integrity": "sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA==", + "dev": true, + "requires": { + "string-width": "^3.1.0", + "strip-ansi": "^5.2.0", + "wrap-ansi": "^5.1.0" + }, + "dependencies": { + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + } + } + }, "coffee-script": { "version": "1.6.0", - "from": "coffee-script@1.6.0", - "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz" + "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz", + "integrity": "sha1-gIs5bhEPU9AhoZpO8fZb4OjjX6M=" + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true }, "combined-stream": { "version": "0.0.7", - "from": "combined-stream@>=0.0.4 <0.1.0", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.7.tgz" + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.7.tgz", + "integrity": "sha1-ATfmV7qlp1QcV6w3rF/AfXO03B8=", + "requires": { + "delayed-stream": "0.0.5" + } + }, + "common-tags": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/common-tags/-/common-tags-1.8.0.tgz", + "integrity": "sha512-6P6g0uetGpW/sdyUy/iQQCbFF0kWVMSIVSyYz7Zgjcgh8mgw8PQzDNZeyZ5DQ2gM7LBoZPHmnjz8rUthkBG5tw==", + "dev": true }, "concat-map": { "version": "0.0.1", - "from": "concat-map@0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, "console-log-level": { "version": "1.4.1", - "from": "console-log-level@>=1.4.0 <2.0.0", - "resolved": "https://registry.npmjs.org/console-log-level/-/console-log-level-1.4.1.tgz" + "resolved": "https://registry.npmjs.org/console-log-level/-/console-log-level-1.4.1.tgz", + "integrity": "sha1-nFprue8e9lsFq6gwKLD/iUzfYwo=" + }, + "contains-path": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz", + "integrity": "sha1-/ozxhP9mcLa67wGp1IYaXL7EEgo=", + "dev": true }, "content-disposition": { "version": "0.5.2", - "from": "content-disposition@0.5.2", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz" + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", + "integrity": "sha1-DPaLud318r55YcOoUXjLhdunjLQ=" }, "content-type": { "version": "1.0.4", - "from": "content-type@>=1.0.4 <1.1.0", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz" + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", + "integrity": "sha1-4TjMdeBAxyexlm/l5fjJruJW/js=" }, "continuation-local-storage": { "version": "3.2.1", - "from": "continuation-local-storage@>=3.2.1 <4.0.0", - "resolved": "https://registry.npmjs.org/continuation-local-storage/-/continuation-local-storage-3.2.1.tgz" + "resolved": "https://registry.npmjs.org/continuation-local-storage/-/continuation-local-storage-3.2.1.tgz", + "integrity": "sha1-EfYT906RT+mzTJKtLSj+auHbf/s=", + "requires": { + "async-listener": "^0.6.0", + "emitter-listener": "^1.1.1" + } }, "cookie": { "version": "0.3.1", - "from": "cookie@0.3.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz" + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz", + "integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s=" }, "cookie-jar": { "version": "0.2.0", - "from": "cookie-jar@>=0.2.0 <0.3.0", - "resolved": "https://registry.npmjs.org/cookie-jar/-/cookie-jar-0.2.0.tgz" + "resolved": "https://registry.npmjs.org/cookie-jar/-/cookie-jar-0.2.0.tgz", + "integrity": "sha1-ZOzAasl423leS1KQy+SLo3gUAPo=" }, "cookie-signature": { "version": "1.0.6", - "from": "cookie-signature@1.0.6", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz" + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" + }, + "core-js": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.5.0.tgz", + "integrity": "sha512-Ifh3kj78gzQ7NAoJXeTu+XwzDld0QRIwjBLRqAMhuLhP3d2Av5wmgE9ycfnvK6NAEjTkQ1sDPeoEZAWO3Hx1Uw==", + "dev": true }, "core-util-is": { "version": "1.0.2", - "from": "core-util-is@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz" + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "cross-spawn": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", + "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==", + "dev": true, + "requires": { + "nice-try": "^1.0.4", + "path-key": "^2.0.1", + "semver": "^5.5.0", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + } }, "cryptiles": { "version": "0.1.3", - "from": "cryptiles@>=0.1.0 <0.2.0", - "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-0.1.3.tgz" + "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-0.1.3.tgz", + "integrity": "sha1-GlVnNPBtJLo0hirpy55wmjr7/xw=", + "requires": { + "boom": "0.3.x" + } }, "dashdash": { "version": "1.14.1", - "from": "dashdash@>=1.12.0 <2.0.0", - "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz" + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "requires": { + "assert-plus": "^1.0.0" + } }, "debug": { "version": "2.6.9", - "from": "debug@2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz" + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha1-XRKFFd8TT/Mn6QpMk/Tgd6U2NB8=", + "requires": { + "ms": "2.0.0" + } + }, + "decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", + "dev": true }, "deep-eql": { "version": "3.0.1", - "from": "deep-eql@>=3.0.1 <4.0.0", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz", + "integrity": "sha1-38lARACtHI/gI+faHfHBR8S0RN8=", + "dev": true, + "requires": { + "type-detect": "^4.0.0" + } + }, + "deep-is": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", + "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", "dev": true }, + "define-properties": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", + "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", + "dev": true, + "requires": { + "object-keys": "^1.0.12" + } + }, "delay": { "version": "4.3.0", - "from": "delay@>=4.0.1 <5.0.0", - "resolved": "https://registry.npmjs.org/delay/-/delay-4.3.0.tgz" + "resolved": "https://registry.npmjs.org/delay/-/delay-4.3.0.tgz", + "integrity": "sha1-7+6/uPVFV5yzlrOnIkQ+yW0UxQ4=" }, "delayed-stream": { "version": "0.0.5", - "from": "delayed-stream@0.0.5", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz" + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz", + "integrity": "sha1-1LH0OpPoKW3+AmlPRoC8N6MTxz8=" }, "depd": { "version": "1.1.2", - "from": "depd@>=1.1.2 <1.2.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz" + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", + "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=" }, "destroy": { "version": "1.0.4", - "from": "destroy@>=1.0.4 <1.1.0", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz" + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", + "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" }, "diff": { "version": "3.5.0", - "from": "diff@>=3.5.0 <4.0.0", "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", + "integrity": "sha1-gAwN0eCov7yVg1wgKtIg/jF+WhI=", "dev": true }, + "dlv": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", + "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==", + "dev": true + }, + "doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dev": true, + "requires": { + "esutils": "^2.0.2" + } + }, "dtrace-provider": { "version": "0.6.0", - "from": "dtrace-provider@>=0.6.0 <0.7.0", "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.6.0.tgz", + "integrity": "sha1-CweNVReTfYcxAUUtkUZzdVe3XlE=", "dev": true, - "optional": true + "optional": true, + "requires": { + "nan": "^2.0.8" + } }, "duplexify": { "version": "3.7.1", - "from": "duplexify@>=3.6.0 <4.0.0", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", + "integrity": "sha1-Kk31MX9sz9kfhtb9JdjYoQO4gwk=", + "requires": { + "end-of-stream": "^1.0.0", + "inherits": "^2.0.1", + "readable-stream": "^2.0.0", + "stream-shift": "^1.0.0" + }, "dependencies": { "readable-stream": { "version": "2.3.6", - "from": "readable-stream@>=2.0.0 <3.0.0", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz" + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha1-sRwn2IuP8fvgcGQ8+UsMea4bCq8=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } }, "string_decoder": { "version": "1.1.1", - "from": "string_decoder@>=1.1.1 <1.2.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz" + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha1-nPFhG6YmhdcDCunkujQUnDrwP8g=", + "requires": { + "safe-buffer": "~5.1.0" + } } } }, "ecc-jsbn": { "version": "0.1.2", - "from": "ecc-jsbn@>=0.1.1 <0.2.0", - "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz" + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "requires": { + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" + } }, "ecdsa-sig-formatter": { "version": "1.0.11", - "from": "ecdsa-sig-formatter@1.0.11", - "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz" + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha1-rg8PothQRe8UqBfao86azQSJ5b8=", + "requires": { + "safe-buffer": "^5.0.1" + } }, "ee-first": { "version": "1.1.1", - "from": "ee-first@1.1.1", - "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz" + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" }, "emitter-listener": { "version": "1.1.2", - "from": "emitter-listener@>=1.1.1 <2.0.0", - "resolved": "https://registry.npmjs.org/emitter-listener/-/emitter-listener-1.1.2.tgz" + "resolved": "https://registry.npmjs.org/emitter-listener/-/emitter-listener-1.1.2.tgz", + "integrity": "sha1-VrFA6PaZI3Wz18ssqxzHQy2WMug=", + "requires": { + "shimmer": "^1.2.0" + } + }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true }, "encodeurl": { "version": "1.0.2", - "from": "encodeurl@>=1.0.2 <1.1.0", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz" + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k=" }, "end-of-stream": { "version": "1.4.1", - "from": "end-of-stream@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz" + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz", + "integrity": "sha1-7SljTRm6ukY7bOa4CjchPqtx7EM=", + "requires": { + "once": "^1.4.0" + } }, "ent": { "version": "2.2.0", - "from": "ent@>=2.2.0 <3.0.0", - "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz" + "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz", + "integrity": "sha1-6WQhkyWiHQX0RGai9obtbOX13R0=" + }, + "error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dev": true, + "requires": { + "is-arrayish": "^0.2.1" + } + }, + "es-abstract": { + "version": "1.17.0-next.1", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.0-next.1.tgz", + "integrity": "sha512-7MmGr03N7Rnuid6+wyhD9sHNE2n4tFSwExnU2lQl3lIo2ShXWGePY80zYaoMOmILWv57H0amMjZGHNzzGG70Rw==", + "dev": true, + "requires": { + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1", + "is-callable": "^1.1.4", + "is-regex": "^1.0.4", + "object-inspect": "^1.7.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.0", + "string.prototype.trimleft": "^2.1.0", + "string.prototype.trimright": "^2.1.0" + } + }, + "es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "dev": true, + "requires": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + } }, "es6-promise": { "version": "4.2.8", - "from": "es6-promise@>=4.0.3 <5.0.0", - "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz" + "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", + "integrity": "sha1-TrIVlMlyvEBVPSduUQU5FD21Pgo=" }, "es6-promisify": { "version": "5.0.0", - "from": "es6-promisify@>=5.0.0 <6.0.0", - "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz" + "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", + "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", + "requires": { + "es6-promise": "^4.0.3" + } }, "escape-html": { "version": "1.0.3", - "from": "escape-html@>=1.0.3 <1.1.0", - "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz" + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" }, "escape-string-regexp": { "version": "1.0.5", - "from": "escape-string-regexp@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz" + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" + }, + "eslint": { + "version": "6.7.2", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-6.7.2.tgz", + "integrity": "sha512-qMlSWJaCSxDFr8fBPvJM9kJwbazrhNcBU3+DszDW1OlEwKBBRWsJc7NJFelvwQpanHCR14cOLD41x8Eqvo3Nng==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "ajv": "^6.10.0", + "chalk": "^2.1.0", + "cross-spawn": "^6.0.5", + "debug": "^4.0.1", + "doctrine": "^3.0.0", + "eslint-scope": "^5.0.0", + "eslint-utils": "^1.4.3", + "eslint-visitor-keys": "^1.1.0", + "espree": "^6.1.2", + "esquery": "^1.0.1", + "esutils": "^2.0.2", + "file-entry-cache": "^5.0.1", + "functional-red-black-tree": "^1.0.1", + "glob-parent": "^5.0.0", + "globals": "^12.1.0", + "ignore": "^4.0.6", + "import-fresh": "^3.0.0", + "imurmurhash": "^0.1.4", + "inquirer": "^7.0.0", + "is-glob": "^4.0.0", + "js-yaml": "^3.13.1", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.3.0", + "lodash": "^4.17.14", + "minimatch": "^3.0.4", + "mkdirp": "^0.5.1", + "natural-compare": "^1.4.0", + "optionator": "^0.8.3", + "progress": "^2.0.0", + "regexpp": "^2.0.1", + "semver": "^6.1.2", + "strip-ansi": "^5.2.0", + "strip-json-comments": "^3.0.1", + "table": "^5.2.3", + "text-table": "^0.2.0", + "v8-compile-cache": "^2.0.3" + } + }, + "eslint-config-prettier": { + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-6.7.0.tgz", + "integrity": "sha512-FamQVKM3jjUVwhG4hEMnbtsq7xOIDm+SY5iBPfR8gKsJoAB2IQnNF+bk1+8Fy44Nq7PPJaLvkRxILYdJWoguKQ==", + "dev": true, + "requires": { + "get-stdin": "^6.0.0" + } + }, + "eslint-config-standard": { + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/eslint-config-standard/-/eslint-config-standard-14.1.0.tgz", + "integrity": "sha512-EF6XkrrGVbvv8hL/kYa/m6vnvmUT+K82pJJc4JJVMM6+Qgqh0pnwprSxdduDLB9p/7bIxD+YV5O0wfb8lmcPbA==", + "dev": true + }, + "eslint-import-resolver-node": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.2.tgz", + "integrity": "sha512-sfmTqJfPSizWu4aymbPr4Iidp5yKm8yDkHp+Ir3YiTHiiDfxh69mOUsmiqW6RZ9zRXFaF64GtYmN7e+8GHBv6Q==", + "dev": true, + "requires": { + "debug": "^2.6.9", + "resolve": "^1.5.0" + } + }, + "eslint-module-utils": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.5.0.tgz", + "integrity": "sha512-kCo8pZaNz2dsAW7nCUjuVoI11EBXXpIzfNxmaoLhXoRDOnqXLC4iSGVRdZPhOitfbdEfMEfKOiENaK6wDPZEGw==", + "dev": true, + "requires": { + "debug": "^2.6.9", + "pkg-dir": "^2.0.0" + } + }, + "eslint-plugin-chai-expect": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-chai-expect/-/eslint-plugin-chai-expect-2.1.0.tgz", + "integrity": "sha512-rd0/4mjMV6c3i0o4DKkWI4uaFN9DK707kW+/fDphaDI6HVgxXnhML9Xgt5vHnTXmSSnDhupuCFBgsEAEpchXmQ==", + "dev": true + }, + "eslint-plugin-chai-friendly": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-chai-friendly/-/eslint-plugin-chai-friendly-0.5.0.tgz", + "integrity": "sha512-Pxe6z8C9fP0pn2X2nGFU/b3GBOCM/5FVus1hsMwJsXP3R7RiXFl7g0ksJbsc0GxiLyidTW4mEFk77qsNn7Tk7g==", + "dev": true + }, + "eslint-plugin-es": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-es/-/eslint-plugin-es-2.0.0.tgz", + "integrity": "sha512-f6fceVtg27BR02EYnBhgWLFQfK6bN4Ll0nQFrBHOlCsAyxeZkn0NHns5O0YZOPrV1B3ramd6cgFwaoFLcSkwEQ==", + "dev": true, + "requires": { + "eslint-utils": "^1.4.2", + "regexpp": "^3.0.0" + }, + "dependencies": { + "regexpp": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.0.0.tgz", + "integrity": "sha512-Z+hNr7RAVWxznLPuA7DIh8UNX1j9CDrUQxskw9IrBE1Dxue2lyXT+shqEIeLUjrokxIP8CMy1WkjgG3rTsd5/g==", + "dev": true + } + } + }, + "eslint-plugin-import": { + "version": "2.19.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.19.1.tgz", + "integrity": "sha512-x68131aKoCZlCae7rDXKSAQmbT5DQuManyXo2sK6fJJ0aK5CWAkv6A6HJZGgqC8IhjQxYPgo6/IY4Oz8AFsbBw==", + "dev": true, + "requires": { + "array-includes": "^3.0.3", + "array.prototype.flat": "^1.2.1", + "contains-path": "^0.1.0", + "debug": "^2.6.9", + "doctrine": "1.5.0", + "eslint-import-resolver-node": "^0.3.2", + "eslint-module-utils": "^2.4.1", + "has": "^1.0.3", + "minimatch": "^3.0.4", + "object.values": "^1.1.0", + "read-pkg-up": "^2.0.0", + "resolve": "^1.12.0" + }, + "dependencies": { + "doctrine": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-1.5.0.tgz", + "integrity": "sha1-N53Ocw9hZvds76TmcHoVmwLFpvo=", + "dev": true, + "requires": { + "esutils": "^2.0.2", + "isarray": "^1.0.0" + } + } + } + }, + "eslint-plugin-mocha": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/eslint-plugin-mocha/-/eslint-plugin-mocha-6.2.2.tgz", + "integrity": "sha512-oNhPzfkT6Q6CJ0HMVJ2KLxEWG97VWGTmuHOoRcDLE0U88ugUyFNV9wrT2XIt5cGtqc5W9k38m4xTN34L09KhBA==", + "dev": true, + "requires": { + "ramda": "^0.26.1" + } + }, + "eslint-plugin-node": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-node/-/eslint-plugin-node-10.0.0.tgz", + "integrity": "sha512-1CSyM/QCjs6PXaT18+zuAXsjXGIGo5Rw630rSKwokSs2jrYURQc4R5JZpoanNCqwNmepg+0eZ9L7YiRUJb8jiQ==", + "dev": true, + "requires": { + "eslint-plugin-es": "^2.0.0", + "eslint-utils": "^1.4.2", + "ignore": "^5.1.1", + "minimatch": "^3.0.4", + "resolve": "^1.10.1", + "semver": "^6.1.0" + }, + "dependencies": { + "ignore": { + "version": "5.1.4", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.4.tgz", + "integrity": "sha512-MzbUSahkTW1u7JpKKjY7LCARd1fU5W2rLdxlM4kdkayuCwZImjkpluF9CM1aLewYJguPDqewLam18Y6AU69A8A==", + "dev": true + } + } + }, + "eslint-plugin-prettier": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-3.1.2.tgz", + "integrity": "sha512-GlolCC9y3XZfv3RQfwGew7NnuFDKsfI4lbvRK+PIIo23SFH+LemGs4cKwzAaRa+Mdb+lQO/STaIayno8T5sJJA==", + "dev": true, + "requires": { + "prettier-linter-helpers": "^1.0.0" + } + }, + "eslint-plugin-promise": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-promise/-/eslint-plugin-promise-4.2.1.tgz", + "integrity": "sha512-VoM09vT7bfA7D+upt+FjeBO5eHIJQBUWki1aPvB+vbNiHS3+oGIJGIeyBtKQTME6UPXXy3vV07OL1tHd3ANuDw==", + "dev": true + }, + "eslint-plugin-standard": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-standard/-/eslint-plugin-standard-4.0.1.tgz", + "integrity": "sha512-v/KBnfyaOMPmZc/dmc6ozOdWqekGp7bBGq4jLAecEfPGmfKiWS4sA8sC0LqiV9w5qmXAtXVn4M3p1jSyhY85SQ==", + "dev": true + }, + "eslint-scope": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.0.0.tgz", + "integrity": "sha512-oYrhJW7S0bxAFDvWqzvMPRm6pcgcnWc4QnofCAqRTRfQC0JcwenzGglTtsLyIuuWFfkqDG9vz67cnttSd53djw==", + "dev": true, + "requires": { + "esrecurse": "^4.1.0", + "estraverse": "^4.1.1" + } + }, + "eslint-utils": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-1.4.3.tgz", + "integrity": "sha512-fbBN5W2xdY45KulGXmLHZ3c3FHfVYmKg0IrAKGOkT/464PQsx2UeIzfz1RmEci+KLm1bBaAzZAh8+/E+XAeZ8Q==", + "dev": true, + "requires": { + "eslint-visitor-keys": "^1.1.0" + } + }, + "eslint-visitor-keys": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.1.0.tgz", + "integrity": "sha512-8y9YjtM1JBJU/A9Kc+SbaOV4y29sSWckBwMHa+FGtVj5gN/sbnKDf6xJUl+8g7FAij9LVaP8C24DUiH/f/2Z9A==", + "dev": true + }, + "espree": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/espree/-/espree-6.1.2.tgz", + "integrity": "sha512-2iUPuuPP+yW1PZaMSDM9eyVf8D5P0Hi8h83YtZ5bPc/zHYjII5khoixIUTMO794NOY8F/ThF1Bo8ncZILarUTA==", + "dev": true, + "requires": { + "acorn": "^7.1.0", + "acorn-jsx": "^5.1.0", + "eslint-visitor-keys": "^1.1.0" + } + }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true + }, + "esquery": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.0.1.tgz", + "integrity": "sha512-SmiyZ5zIWH9VM+SRUReLS5Q8a7GxtRdxEBVZpm98rJM7Sb+A9DVCndXfkeFUd3byderg+EbDkfnevfCwynWaNA==", + "dev": true, + "requires": { + "estraverse": "^4.0.0" + } + }, + "esrecurse": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz", + "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==", + "dev": true, + "requires": { + "estraverse": "^4.1.0" + } + }, + "estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "dev": true + }, + "esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true }, "etag": { "version": "1.8.1", - "from": "etag@>=1.8.1 <1.9.0", - "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz" + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" }, "event-target-shim": { "version": "5.0.1", - "from": "event-target-shim@>=5.0.0 <6.0.0", - "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz" + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha1-XU0+vflYPWOlMzzi3rdICrKwV4k=" }, "events": { "version": "1.1.1", - "from": "events@1.1.1", - "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz" + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" }, "express": { "version": "4.16.3", - "from": "express@>=4.2.0 <5.0.0", "resolved": "https://registry.npmjs.org/express/-/express-4.16.3.tgz", + "integrity": "sha1-avilAjUNsyRuzEvs9rWjTSL37VM=", + "requires": { + "accepts": "~1.3.5", + "array-flatten": "1.1.1", + "body-parser": "1.18.2", + "content-disposition": "0.5.2", + "content-type": "~1.0.4", + "cookie": "0.3.1", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "~1.1.2", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.1.1", + "fresh": "0.5.2", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "~2.3.0", + "parseurl": "~1.3.2", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.3", + "qs": "6.5.1", + "range-parser": "~1.2.0", + "safe-buffer": "5.1.1", + "send": "0.16.2", + "serve-static": "1.13.2", + "setprototypeof": "1.1.0", + "statuses": "~1.4.0", + "type-is": "~1.6.16", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, "dependencies": { "body-parser": { "version": "1.18.2", - "from": "body-parser@1.18.2", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz" + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz", + "integrity": "sha1-h2eKGdhLR9hZuDGZvVm84iKxBFQ=", + "requires": { + "bytes": "3.0.0", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "~1.1.1", + "http-errors": "~1.6.2", + "iconv-lite": "0.4.19", + "on-finished": "~2.3.0", + "qs": "6.5.1", + "raw-body": "2.3.2", + "type-is": "~1.6.15" + } }, "iconv-lite": { "version": "0.4.19", - "from": "iconv-lite@0.4.19", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz" + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz", + "integrity": "sha1-90aPYBNfXl2tM5nAqBvpoWA6CCs=" }, "qs": { "version": "6.5.1", - "from": "qs@6.5.1", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz" + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", + "integrity": "sha1-NJzfbu+J7EXBLX1es/wMhwNDptg=" }, "raw-body": { "version": "2.3.2", - "from": "raw-body@2.3.2", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.2.tgz", + "integrity": "sha1-vNYMd9Prk83gBQKVw/N5OJvIj4k=", + "requires": { + "bytes": "3.0.0", + "http-errors": "1.6.2", + "iconv-lite": "0.4.19", + "unpipe": "1.0.0" + }, "dependencies": { "depd": { "version": "1.1.1", - "from": "depd@1.1.1", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.1.tgz" + "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.1.tgz", + "integrity": "sha1-V4O04cRZ8G+lyif5kfPQbnoxA1k=" }, "http-errors": { "version": "1.6.2", - "from": "http-errors@1.6.2", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.2.tgz" + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.2.tgz", + "integrity": "sha1-CgAsyFcHGSp+eUbO7cERVfYOxzY=", + "requires": { + "depd": "1.1.1", + "inherits": "2.0.3", + "setprototypeof": "1.0.3", + "statuses": ">= 1.3.1 < 2" + } }, "setprototypeof": { "version": "1.0.3", - "from": "setprototypeof@1.0.3", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz" + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz", + "integrity": "sha1-ZlZ+NwQ+608E2RvWWMDL77VbjgQ=" } } }, "statuses": { "version": "1.4.0", - "from": "statuses@>=1.4.0 <1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz" + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", + "integrity": "sha1-u3PURtonlhBu/MG2AaJT1sRr0Ic=" } } }, "extend": { "version": "3.0.2", - "from": "extend@>=3.0.2 <3.1.0", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz" + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha1-+LETa0Bx+9jrFAr/hYsQGewpFfo=" + }, + "external-editor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", + "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==", + "dev": true, + "requires": { + "chardet": "^0.7.0", + "iconv-lite": "^0.4.24", + "tmp": "^0.0.33" + } }, "extsprintf": { "version": "1.3.0", - "from": "extsprintf@1.3.0", - "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz" + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" }, "fast-deep-equal": { "version": "2.0.1", - "from": "fast-deep-equal@>=2.0.1 <3.0.0", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz" + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" + }, + "fast-diff": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.2.0.tgz", + "integrity": "sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==", + "dev": true }, "fast-json-stable-stringify": { "version": "2.0.0", - "from": "fast-json-stable-stringify@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz" + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", + "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" + }, + "fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", + "dev": true }, "fast-text-encoding": { "version": "1.0.0", - "from": "fast-text-encoding@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.0.tgz", + "integrity": "sha1-PlzoKTQJz6pxd6cbnKhOGx5vJe8=" + }, + "figures": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-3.1.0.tgz", + "integrity": "sha512-ravh8VRXqHuMvZt/d8GblBeqDMkdJMBdv/2KntFH+ra5MXkO7nxNKpzQ3n6QD/2da1kH0aWmNISdvhM7gl2gVg==", + "dev": true, + "requires": { + "escape-string-regexp": "^1.0.5" + } + }, + "file-entry-cache": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-5.0.1.tgz", + "integrity": "sha512-bCg29ictuBaKUwwArK4ouCaqDgLZcysCFLmM/Yn/FDoqndh/9vNuQfXRDvTuXKLxfD/JtZQGKFT8MGcJBK644g==", + "dev": true, + "requires": { + "flat-cache": "^2.0.1" + } }, "file-uri-to-path": { "version": "1.0.0", - "from": "file-uri-to-path@1.0.0", - "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha1-VTp7hEb/b2hDWcRF8eN6BdrMM90=" }, "finalhandler": { "version": "1.1.1", - "from": "finalhandler@1.1.1", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", + "integrity": "sha1-7r9O2EAHnIP0JJA4ydcDAIMBsQU=", + "requires": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "~2.3.0", + "parseurl": "~1.3.2", + "statuses": "~1.4.0", + "unpipe": "~1.0.0" + }, "dependencies": { "statuses": { "version": "1.4.0", - "from": "statuses@>=1.4.0 <1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz" + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", + "integrity": "sha1-u3PURtonlhBu/MG2AaJT1sRr0Ic=" } } }, + "find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", + "dev": true, + "requires": { + "locate-path": "^2.0.0" + } + }, "findit2": { "version": "2.2.3", - "from": "findit2@>=2.2.3 <3.0.0", - "resolved": "https://registry.npmjs.org/findit2/-/findit2-2.2.3.tgz" + "resolved": "https://registry.npmjs.org/findit2/-/findit2-2.2.3.tgz", + "integrity": "sha1-WKRmaX34piBc39vzlVNri9d3pfY=" + }, + "flat-cache": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-2.0.1.tgz", + "integrity": "sha512-LoQe6yDuUMDzQAEH8sgmh4Md6oZnc/7PjtwjNFSzveXqSHt6ka9fPBuso7IGf9Rz4uqnSnWiFH2B/zj24a5ReA==", + "dev": true, + "requires": { + "flatted": "^2.0.0", + "rimraf": "2.6.3", + "write": "1.0.3" + } + }, + "flatted": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.1.tgz", + "integrity": "sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg==", + "dev": true }, "follow-redirects": { "version": "1.5.10", - "from": "follow-redirects@1.5.10", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz", + "integrity": "sha1-e3qfmuov3/NnhqlP9kPtB/T/Xio=", + "requires": { + "debug": "=3.1.0" + }, "dependencies": { "debug": { "version": "3.1.0", - "from": "debug@3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz" + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha1-W7WgZyYotkFJVmuhaBnmFRjGcmE=", + "requires": { + "ms": "2.0.0" + } } } }, "forever-agent": { "version": "0.2.0", - "from": "forever-agent@>=0.2.0 <0.3.0", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.2.0.tgz" + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.2.0.tgz", + "integrity": "sha1-4cJcetROCcOPIzh2x2/MJP+EOx8=" }, "form-data": { "version": "0.0.10", - "from": "form-data@>=0.0.3 <0.1.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-0.0.10.tgz", + "integrity": "sha1-2zRaU3jYau6x7V1VO4aawZLS9e0=", + "requires": { + "async": "~0.2.7", + "combined-stream": "~0.0.4", + "mime": "~1.2.2" + }, "dependencies": { "mime": { "version": "1.2.11", - "from": "mime@>=1.2.2 <1.3.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz" + "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz", + "integrity": "sha1-WCA+7Ybjpe8XrtK32evUfwpg3RA=" } } }, "forwarded": { "version": "0.1.2", - "from": "forwarded@>=0.1.2 <0.2.0", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz" + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", + "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" }, "fresh": { "version": "0.5.2", - "from": "fresh@0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz" + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, "fs-extra": { "version": "1.0.0", - "from": "fs-extra@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz", + "integrity": "sha1-zTzl9+fLYUWIP8rjGR6Yd/hYeVA=", + "requires": { + "graceful-fs": "^4.1.2", + "jsonfile": "^2.1.0", + "klaw": "^1.0.0" + } }, "fs.realpath": { "version": "1.0.0", - "from": "fs.realpath@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "dev": true + }, + "functional-red-black-tree": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", + "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", + "dev": true }, "gaxios": { "version": "1.8.4", - "from": "gaxios@>=1.2.1 <2.0.0", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.8.4.tgz" + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.8.4.tgz", + "integrity": "sha1-4Iw0/pPAqbZ6Ure556ZOZDX5ozk=", + "requires": { + "abort-controller": "^3.0.0", + "extend": "^3.0.2", + "https-proxy-agent": "^2.2.1", + "node-fetch": "^2.3.0" + } }, "gcp-metadata": { "version": "1.0.0", - "from": "gcp-metadata@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-1.0.0.tgz", + "integrity": "sha1-UhJEAin6CZ/C98KlzcuVV16bLKY=", + "requires": { + "gaxios": "^1.0.2", + "json-bigint": "^0.3.0" + } + }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true }, "get-func-name": { "version": "2.0.0", - "from": "get-func-name@>=2.0.0 <3.0.0", "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", + "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=", + "dev": true + }, + "get-stdin": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-6.0.0.tgz", + "integrity": "sha512-jp4tHawyV7+fkkSKyvjuLZswblUtz+SQKzSWnBbii16BuZksJlU1wuBYXY75r+duh/llF1ur6oNwi+2ZzjKZ7g==", "dev": true }, "getpass": { "version": "0.1.7", - "from": "getpass@>=0.1.1 <0.2.0", - "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz" + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "requires": { + "assert-plus": "^1.0.0" + } }, "gettemporaryfilepath": { "version": "0.0.1", - "from": "gettemporaryfilepath@0.0.1", - "resolved": "https://registry.npmjs.org/gettemporaryfilepath/-/gettemporaryfilepath-0.0.1.tgz" + "resolved": "https://registry.npmjs.org/gettemporaryfilepath/-/gettemporaryfilepath-0.0.1.tgz", + "integrity": "sha1-uKLHAUu1zUFTTpg7XKFgo3RwhGk=" }, "glob": { "version": "6.0.4", - "from": "glob@>=6.0.1 <7.0.0", "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", - "optional": true + "integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=", + "requires": { + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "2 || 3", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "glob-parent": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.0.tgz", + "integrity": "sha512-qjtRgnIVmOfnKUE3NJAQEdk+lKrxfw8t5ke7SXtfMTHcjsBfOfWXCQfdb30zfDoZQ2IRSIiidmjtbHZPZ++Ihw==", + "dev": true, + "requires": { + "is-glob": "^4.0.1" + } + }, + "globals": { + "version": "12.3.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-12.3.0.tgz", + "integrity": "sha512-wAfjdLgFsPZsklLJvOBUBmzYE8/CwhEqSBEMRXA3qxIiNtyqvjYurAtIfDh6chlEPUfmTY3MnZh5Hfh4q0UlIw==", + "dev": true, + "requires": { + "type-fest": "^0.8.1" + } }, "google-auth-library": { "version": "3.1.2", - "from": "google-auth-library@>=3.1.1 <4.0.0", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-3.1.2.tgz", + "integrity": "sha1-/y+IzVzSEYpXvT1a08CTyIN/w1A=", + "requires": { + "base64-js": "^1.3.0", + "fast-text-encoding": "^1.0.0", + "gaxios": "^1.2.1", + "gcp-metadata": "^1.0.0", + "gtoken": "^2.3.2", + "https-proxy-agent": "^2.2.1", + "jws": "^3.1.5", + "lru-cache": "^5.0.0", + "semver": "^5.5.0" + }, "dependencies": { "semver": { "version": "5.7.0", - "from": "semver@>=5.5.0 <6.0.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz" + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", + "integrity": "sha1-eQp89v6lRZuslhELKbYEEtyP+Ws=" } } }, "google-p12-pem": { "version": "1.0.4", - "from": "google-p12-pem@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.4.tgz" + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.4.tgz", + "integrity": "sha1-t3+4M6Lrn388aJ4uVPCVJ293dgU=", + "requires": { + "node-forge": "^0.8.0", + "pify": "^4.0.0" + } }, "graceful-fs": { "version": "4.1.11", - "from": "graceful-fs@>=4.1.2 <5.0.0", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz" + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz", + "integrity": "sha1-Dovf5NHduIVNZOBOp8AOKgJuVlg=" }, "gtoken": { "version": "2.3.3", - "from": "gtoken@>=2.3.2 <3.0.0", "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-2.3.3.tgz", + "integrity": "sha1-in/hVcXODEtxyIbPsoKpBg2UpkE=", + "requires": { + "gaxios": "^1.0.4", + "google-p12-pem": "^1.0.0", + "jws": "^3.1.5", + "mime": "^2.2.0", + "pify": "^4.0.0" + }, "dependencies": { "mime": { "version": "2.4.4", - "from": "mime@>=2.2.0 <3.0.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz" + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz", + "integrity": "sha1-vXuRE1/GsBzePpuuM9ZZtj2IV+U=" } } }, "har-schema": { "version": "2.0.0", - "from": "har-schema@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz" + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" }, "har-validator": { "version": "5.1.3", - "from": "har-validator@>=5.1.0 <5.2.0", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz" + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha1-HvievT5JllV2de7ZiTEQ3DUPoIA=", + "requires": { + "ajv": "^6.5.5", + "har-schema": "^2.0.0" + } + }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "requires": { + "function-bind": "^1.1.1" + } + }, + "has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", + "dev": true, + "requires": { + "ansi-regex": "^2.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "dev": true + } + } }, "has-flag": { "version": "3.0.0", - "from": "has-flag@>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz" + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" + }, + "has-symbols": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", + "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", + "dev": true }, "hawk": { "version": "0.10.2", - "from": "hawk@>=0.10.2 <0.11.0", - "resolved": "https://registry.npmjs.org/hawk/-/hawk-0.10.2.tgz" + "resolved": "https://registry.npmjs.org/hawk/-/hawk-0.10.2.tgz", + "integrity": "sha1-mzYd7pWpMWQObVBOBWCaj8OsRdI=", + "requires": { + "boom": "0.3.x", + "cryptiles": "0.1.x", + "hoek": "0.7.x", + "sntp": "0.1.x" + } }, "he": { "version": "1.1.1", - "from": "he@1.1.1", - "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz" + "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", + "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=" }, "heapdump": { "version": "0.3.9", - "from": "heapdump@>=0.3.2 <0.4.0", - "resolved": "https://registry.npmjs.org/heapdump/-/heapdump-0.3.9.tgz" + "resolved": "https://registry.npmjs.org/heapdump/-/heapdump-0.3.9.tgz", + "integrity": "sha1-A8dOsN9dZ74Jgug0KbqcnSs7f3g=" }, "hex2dec": { "version": "1.1.2", - "from": "hex2dec@>=1.0.1 <2.0.0", - "resolved": "https://registry.npmjs.org/hex2dec/-/hex2dec-1.1.2.tgz" + "resolved": "https://registry.npmjs.org/hex2dec/-/hex2dec-1.1.2.tgz", + "integrity": "sha1-jhzkvvNqdPfVcjw/swkMKGAHczg=" }, "hoek": { "version": "0.7.6", - "from": "hoek@>=0.7.0 <0.8.0", - "resolved": "https://registry.npmjs.org/hoek/-/hoek-0.7.6.tgz" + "resolved": "https://registry.npmjs.org/hoek/-/hoek-0.7.6.tgz", + "integrity": "sha1-YPvZBFV1Qc0rh5Wr8wihs3cOFVo=" + }, + "hosted-git-info": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.5.tgz", + "integrity": "sha512-kssjab8CvdXfcXMXVcvsXum4Hwdq9XGtRD3TteMEvEbq0LXyiNQr6AprqKqfeaDXze7SxWvRxdpwE6ku7ikLkg==", + "dev": true }, "http-errors": { "version": "1.6.3", - "from": "http-errors@>=1.6.3 <1.7.0", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz" + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", + "requires": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.0", + "statuses": ">= 1.4.0 < 2" + } }, "http-signature": { "version": "1.2.0", - "from": "http-signature@>=1.2.0 <1.3.0", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz" + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "requires": { + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + } }, "https-proxy-agent": { "version": "2.2.1", - "from": "https-proxy-agent@>=2.2.1 <3.0.0", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz", + "integrity": "sha1-UVUpcPoE1yPgTFbQQXjD+SWSu8A=", + "requires": { + "agent-base": "^4.1.0", + "debug": "^3.1.0" + }, "dependencies": { "debug": { "version": "3.2.6", - "from": "debug@>=3.1.0 <4.0.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz" + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha1-6D0X3hbYp++3cX7b5fsQE17uYps=", + "requires": { + "ms": "^2.1.1" + } }, "ms": { "version": "2.1.2", - "from": "ms@>=2.1.1 <3.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha1-0J0fNXtEP0kzgqjrPM0YOHKuYAk=" } } }, "iconv-lite": { "version": "0.4.23", - "from": "iconv-lite@0.4.23", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz" + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.23.tgz", + "integrity": "sha1-KXhx9jvlB63Pv8pxXQzQ7thOmmM=", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } }, "ieee754": { "version": "1.1.8", - "from": "ieee754@1.1.8", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.8.tgz" + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.8.tgz", + "integrity": "sha1-vjPUCsEO8ZJnAfbwii2G+/0a0+Q=" + }, + "ignore": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", + "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "dev": true + }, + "import-fresh": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", + "integrity": "sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ==", + "dev": true, + "requires": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + } + }, + "imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", + "dev": true + }, + "indent-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "dev": true }, "inflight": { "version": "1.0.6", - "from": "inflight@>=1.0.4 <2.0.0", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz" + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } }, "inherits": { "version": "2.0.3", - "from": "inherits@2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz" + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" + }, + "inquirer": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-7.0.1.tgz", + "integrity": "sha512-V1FFQ3TIO15det8PijPLFR9M9baSlnRs9nL7zWu1MNVA2T9YVl9ZbrHJhYs7e9X8jeMZ3lr2JH/rdHFgNCBdYw==", + "dev": true, + "requires": { + "ansi-escapes": "^4.2.1", + "chalk": "^2.4.2", + "cli-cursor": "^3.1.0", + "cli-width": "^2.0.0", + "external-editor": "^3.0.3", + "figures": "^3.0.0", + "lodash": "^4.17.15", + "mute-stream": "0.0.8", + "run-async": "^2.2.0", + "rxjs": "^6.5.3", + "string-width": "^4.1.0", + "strip-ansi": "^5.1.0", + "through": "^2.3.6" + } }, "ipaddr.js": { "version": "1.8.0", - "from": "ipaddr.js@1.8.0", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.8.0.tgz" + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.8.0.tgz", + "integrity": "sha1-6qM9bd16zo9/b+DJygRA5wZzix4=" }, "is": { "version": "3.3.0", - "from": "is@>=3.2.0 <4.0.0", - "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz" + "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz", + "integrity": "sha1-Yc/23TxBk9uUo9YlggcrROVkXXk=" + }, + "is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=", + "dev": true }, "is-buffer": { "version": "2.0.3", - "from": "is-buffer@>=2.0.2 <3.0.0", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.3.tgz" + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.3.tgz", + "integrity": "sha1-Ts8/z3ScvR5HJonhCaxmJhol5yU=" + }, + "is-callable": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.4.tgz", + "integrity": "sha512-r5p9sxJjYnArLjObpjA4xu5EKI3CuKHkJXMhT7kwbpUyIFD1n5PMAsoPvWnvtZiNz7LjkYDRZhd7FlI0eMijEA==", + "dev": true + }, + "is-date-object": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz", + "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY=", + "dev": true + }, + "is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "is-glob": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz", + "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==", + "dev": true, + "requires": { + "is-extglob": "^2.1.1" + } + }, + "is-promise": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz", + "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=", + "dev": true + }, + "is-regex": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.5.tgz", + "integrity": "sha512-vlKW17SNq44owv5AQR3Cq0bQPEb8+kF3UKZ2fiZNOWtztYE5i0CzCZxFDwO58qAOWtxdBRVO/V5Qin1wjCqFYQ==", + "dev": true, + "requires": { + "has": "^1.0.3" + } + }, + "is-symbol": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz", + "integrity": "sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==", + "dev": true, + "requires": { + "has-symbols": "^1.0.1" + } }, "is-typedarray": { "version": "1.0.0", - "from": "is-typedarray@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" }, "isarray": { "version": "1.0.0", - "from": "isarray@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", + "dev": true }, "isstream": { "version": "0.1.2", - "from": "isstream@>=0.1.0 <0.2.0", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz" + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, "jmespath": { "version": "0.15.0", - "from": "jmespath@0.15.0", - "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz" + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + }, + "js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, + "js-yaml": { + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", + "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } }, "jsbn": { "version": "0.1.1", - "from": "jsbn@>=0.1.0 <0.2.0", - "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz" + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" + }, + "jsesc": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", + "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", + "dev": true }, "json-bigint": { "version": "0.3.0", - "from": "json-bigint@>=0.3.0 <0.4.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.3.0.tgz" + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.3.0.tgz", + "integrity": "sha1-DM2RLEuCcNBfBW+9E4FLU9OCWx4=", + "requires": { + "bignumber.js": "^7.0.0" + } }, "json-schema": { "version": "0.2.3", - "from": "json-schema@0.2.3", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz" + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" }, "json-schema-traverse": { "version": "0.4.1", - "from": "json-schema-traverse@>=0.4.1 <0.5.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz" + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha1-afaofZUTq4u4/mO9sJecRI5oRmA=" + }, + "json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", + "dev": true }, "json-stringify-safe": { "version": "5.0.1", - "from": "json-stringify-safe@5.0.1", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz" + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" }, "jsonfile": { "version": "2.4.0", - "from": "jsonfile@>=2.1.0 <3.0.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz" + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", + "integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=", + "requires": { + "graceful-fs": "^4.1.6" + } }, "jsprim": { "version": "1.4.1", - "from": "jsprim@>=1.2.2 <2.0.0", - "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz" + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "requires": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + } }, "just-extend": { "version": "4.0.2", - "from": "just-extend@>=4.0.2 <5.0.0", "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.0.2.tgz", + "integrity": "sha1-8/R/ffyg+YnFVBCn68iFSwcQivw=", "dev": true }, "jwa": { "version": "1.4.1", - "from": "jwa@>=1.4.1 <2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz" + "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", + "integrity": "sha1-dDwymFy56YZVUw1TZBtmyGRbA5o=", + "requires": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } }, "jws": { "version": "3.2.2", - "from": "jws@>=3.1.5 <4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz" + "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", + "integrity": "sha1-ABCZ82OUaMlBQADpmZX6UvtHgwQ=", + "requires": { + "jwa": "^1.4.1", + "safe-buffer": "^5.0.1" + } }, "klaw": { "version": "1.3.1", - "from": "klaw@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/klaw/-/klaw-1.3.1.tgz" + "resolved": "https://registry.npmjs.org/klaw/-/klaw-1.3.1.tgz", + "integrity": "sha1-QIhDO0azsbolnXh4XY6W9zugJDk=", + "requires": { + "graceful-fs": "^4.1.9" + } }, "knox": { "version": "0.9.2", - "from": "knox@>=0.9.1 <0.10.0", "resolved": "https://registry.npmjs.org/knox/-/knox-0.9.2.tgz", + "integrity": "sha1-NzZZNmniTwJP2vcjtqHcSv2DmnE=", + "requires": { + "debug": "^1.0.2", + "mime": "*", + "once": "^1.3.0", + "stream-counter": "^1.0.0", + "xml2js": "^0.4.4" + }, "dependencies": { "debug": { "version": "1.0.5", - "from": "debug@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-1.0.5.tgz" + "resolved": "https://registry.npmjs.org/debug/-/debug-1.0.5.tgz", + "integrity": "sha1-9yQSF0MPmd7EwrRz6rkiKOh0wqw=", + "requires": { + "ms": "2.0.0" + } } } }, + "levn": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", + "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", + "dev": true, + "requires": { + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2" + } + }, + "load-json-file": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz", + "integrity": "sha1-eUfkIUmvgNaWy/eXvKq8/h/inKg=", + "dev": true, + "requires": { + "graceful-fs": "^4.1.2", + "parse-json": "^2.2.0", + "pify": "^2.0.0", + "strip-bom": "^3.0.0" + } + }, + "locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", + "dev": true, + "requires": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + } + }, "lodash": { "version": "4.17.11", - "from": "lodash@>=4.17.11 <5.0.0", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", + "integrity": "sha1-s56mIp72B+zYniyN8SU2iRysm40=", "dev": true }, "lodash.get": { "version": "4.4.2", - "from": "lodash.get@>=4.4.2 <5.0.0", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=", + "dev": true + }, + "lodash.memoize": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", + "integrity": "sha1-vMbEmkKihA7Zl/Mj6tpezRguC/4=", + "dev": true + }, + "lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, "lodash.pickby": { "version": "4.6.0", - "from": "lodash.pickby@>=4.6.0 <5.0.0", - "resolved": "https://registry.npmjs.org/lodash.pickby/-/lodash.pickby-4.6.0.tgz" + "resolved": "https://registry.npmjs.org/lodash.pickby/-/lodash.pickby-4.6.0.tgz", + "integrity": "sha1-feoh2MGNdwOifHBMFdO4SmfjOv8=" + }, + "lodash.unescape": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.unescape/-/lodash.unescape-4.0.1.tgz", + "integrity": "sha1-vyJJiGzlFM2hEvrpIYzcBlIR/Jw=", + "dev": true }, "logger-sharelatex": { "version": "1.7.0", - "from": "logger-sharelatex@1.7.0", "resolved": "https://registry.npmjs.org/logger-sharelatex/-/logger-sharelatex-1.7.0.tgz", + "integrity": "sha1-XuMje84im1rITZ7SLoXa6eI3/HQ=", + "requires": { + "bunyan": "1.8.12", + "raven": "1.1.3", + "request": "2.88.0" + }, "dependencies": { "bunyan": { "version": "1.8.12", - "from": "bunyan@1.8.12", - "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.12.tgz" + "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.12.tgz", + "integrity": "sha1-8VDw9nSKvdcq6uhPBEA74u8RN5c=", + "requires": { + "dtrace-provider": "~0.8", + "moment": "^2.10.6", + "mv": "~2", + "safe-json-stringify": "~1" + } }, "caseless": { "version": "0.12.0", - "from": "caseless@>=0.12.0 <0.13.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz" + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" }, "combined-stream": { "version": "1.0.8", - "from": "combined-stream@>=1.0.6 <1.1.0", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz" + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha1-w9RaizT9cwYxoRCoolIGgrMdWn8=", + "requires": { + "delayed-stream": "~1.0.0" + } }, "delayed-stream": { "version": "1.0.0", - "from": "delayed-stream@>=1.0.0 <1.1.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" }, "dtrace-provider": { "version": "0.8.7", - "from": "dtrace-provider@>=0.8.0 <0.9.0", "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.7.tgz", - "optional": true + "integrity": "sha1-3JObTT4GIM/gwc2APQ0tftBP/QQ=", + "optional": true, + "requires": { + "nan": "^2.10.0" + } }, "forever-agent": { "version": "0.6.1", - "from": "forever-agent@>=0.6.1 <0.7.0", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz" + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" }, "form-data": { "version": "2.3.3", - "from": "form-data@>=2.3.2 <2.4.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz" + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha1-3M5SwF9kTymManq5Nr1yTO/786Y=", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } }, "oauth-sign": { "version": "0.9.0", - "from": "oauth-sign@>=0.9.0 <0.10.0", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz" + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha1-R6ewFrqmi1+g7PPe4IqFxnmsZFU=" }, "request": { "version": "2.88.0", - "from": "request@>=2.88.0 <3.0.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz" + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha1-nC/KT301tZLv5Xx/ClXoEFIST+8=", + "requires": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.0", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.4.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + } }, "safe-buffer": { "version": "5.1.2", - "from": "safe-buffer@>=5.1.2 <6.0.0", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz" + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha1-mR7GnSluAxN0fVm9/St0XDX4go0=" }, "tunnel-agent": { "version": "0.6.0", - "from": "tunnel-agent@>=0.6.0 <0.7.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz" + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "requires": { + "safe-buffer": "^5.0.1" + } }, "uuid": { "version": "3.3.2", - "from": "uuid@>=3.3.2 <4.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz" + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha1-G0r0lV6zB3xQHCOHL8ZROBFYcTE=" + } + } + }, + "loglevel": { + "version": "1.6.6", + "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.6.6.tgz", + "integrity": "sha512-Sgr5lbboAUBo3eXCSPL4/KoVz3ROKquOjcctxmHIt+vol2DrqTQe3SwkKKuYhEiWB5kYa13YyopJ69deJ1irzQ==", + "dev": true + }, + "loglevel-colored-level-prefix": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/loglevel-colored-level-prefix/-/loglevel-colored-level-prefix-1.0.0.tgz", + "integrity": "sha1-akAhj9x64V/HbD0PPmdsRlOIYD4=", + "dev": true, + "requires": { + "chalk": "^1.1.3", + "loglevel": "^1.4.1" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "dev": true + }, + "ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=", + "dev": true + }, + "chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", + "dev": true, + "requires": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "dev": true, + "requires": { + "ansi-regex": "^2.0.0" + } } } }, "lolex": { "version": "3.1.0", - "from": "lolex@>=3.0.0 <4.0.0", "resolved": "https://registry.npmjs.org/lolex/-/lolex-3.1.0.tgz", + "integrity": "sha1-Gn/rL+/XWz46f3nw4RDZR24pRDQ=", "dev": true }, "long": { "version": "4.0.0", - "from": "long@>=4.0.0 <5.0.0", - "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz" + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha1-mntxz7fTYaGU6lVSQckvdGjVvyg=" }, "lru-cache": { "version": "5.1.1", - "from": "lru-cache@>=5.0.0 <6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz" + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha1-HaJ+ZxAnGUdpXa9oSOhH8B2EuSA=", + "requires": { + "yallist": "^3.0.2" + } }, "lsmod": { "version": "1.0.0", - "from": "lsmod@1.0.0", - "resolved": "https://registry.npmjs.org/lsmod/-/lsmod-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/lsmod/-/lsmod-1.0.0.tgz", + "integrity": "sha1-mgD3bco26yP6BTUK/htYXUKZ5ks=" }, "lynx": { "version": "0.1.1", - "from": "lynx@>=0.1.1 <0.2.0", - "resolved": "https://registry.npmjs.org/lynx/-/lynx-0.1.1.tgz" + "resolved": "https://registry.npmjs.org/lynx/-/lynx-0.1.1.tgz", + "integrity": "sha1-Mxjc7xaQi4KG6Bisz9sxzXQkj50=", + "requires": { + "mersenne": "~0.0.3", + "statsd-parser": "~0.0.4" + } + }, + "make-plural": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/make-plural/-/make-plural-4.3.0.tgz", + "integrity": "sha512-xTYd4JVHpSCW+aqDof6w/MebaMVNTVYBZhbB/vi513xXdiPT92JMVCo0Jq8W2UZnzYRFeVbQiQ+I25l13JuKvA==", + "dev": true, + "requires": { + "minimist": "^1.2.0" + } + }, + "map-obj": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-4.1.0.tgz", + "integrity": "sha512-glc9y00wgtwcDmp7GaE/0b0OnxpNJsVf3ael/An6Fe2Q51LLwN1er6sdomLRzz5h0+yMpiYLhWYF5R7HeqVd4g==", + "dev": true }, "media-typer": { "version": "0.3.0", - "from": "media-typer@0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz" + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" }, "merge-descriptors": { "version": "1.0.1", - "from": "merge-descriptors@1.0.1", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz" + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" }, "mersenne": { "version": "0.0.4", - "from": "mersenne@>=0.0.3 <0.1.0", - "resolved": "https://registry.npmjs.org/mersenne/-/mersenne-0.0.4.tgz" + "resolved": "https://registry.npmjs.org/mersenne/-/mersenne-0.0.4.tgz", + "integrity": "sha1-QB/ex+whzbngPNPTAhOY2iGycIU=" + }, + "messageformat": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/messageformat/-/messageformat-2.3.0.tgz", + "integrity": "sha512-uTzvsv0lTeQxYI2y1NPa1lItL5VRI8Gb93Y2K2ue5gBPyrbJxfDi/EYWxh2PKv5yO42AJeeqblS9MJSh/IEk4w==", + "dev": true, + "requires": { + "make-plural": "^4.3.0", + "messageformat-formatters": "^2.0.1", + "messageformat-parser": "^4.1.2" + } + }, + "messageformat-formatters": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/messageformat-formatters/-/messageformat-formatters-2.0.1.tgz", + "integrity": "sha512-E/lQRXhtHwGuiQjI7qxkLp8AHbMD5r2217XNe/SREbBlSawe0lOqsFb7rflZJmlQFSULNLIqlcjjsCPlB3m3Mg==", + "dev": true + }, + "messageformat-parser": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/messageformat-parser/-/messageformat-parser-4.1.2.tgz", + "integrity": "sha512-7dWuifeyldz7vhEuL96Kwq1fhZXBW+TUfbnHN4UCrCxoXQTYjHnR78eI66Gk9LaLLsAvzPNVJBaa66DRfFNaiA==", + "dev": true }, "methods": { "version": "1.1.2", - "from": "methods@>=1.1.2 <1.2.0", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz" + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" }, "metrics-sharelatex": { "version": "2.2.0", - "from": "metrics-sharelatex@2.2.0", "resolved": "https://registry.npmjs.org/metrics-sharelatex/-/metrics-sharelatex-2.2.0.tgz", + "integrity": "sha1-RM9oy9FuUQYgfrZ+PvkAhaQWwqk=", + "requires": { + "@google-cloud/debug-agent": "^3.0.0", + "@google-cloud/profiler": "^0.2.3", + "@google-cloud/trace-agent": "^3.2.0", + "coffee-script": "1.6.0", + "lynx": "~0.1.1", + "prom-client": "^11.1.3", + "underscore": "~1.6.0" + }, "dependencies": { "underscore": { "version": "1.6.0", - "from": "underscore@>=1.6.0 <1.7.0", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz" + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz", + "integrity": "sha1-izixDKze9jM3uLJOT/htRa6lKag=" } } }, "mime": { "version": "1.4.1", - "from": "mime@1.4.1", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.4.1.tgz" + "resolved": "https://registry.npmjs.org/mime/-/mime-1.4.1.tgz", + "integrity": "sha1-Eh+evEnjdm8xGnbh+hyAA8SwOqY=" }, "mime-db": { "version": "1.36.0", - "from": "mime-db@>=1.36.0 <1.37.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.36.0.tgz" + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.36.0.tgz", + "integrity": "sha1-UCBHjbPH/pOq17vMTc+GnEM2M5c=" }, "mime-types": { "version": "2.1.20", - "from": "mime-types@>=2.1.18 <2.2.0", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.20.tgz" + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.20.tgz", + "integrity": "sha1-kwy3GdVx6QNzhSD4RwkRVIyizBk=", + "requires": { + "mime-db": "~1.36.0" + } + }, + "mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true }, "minimatch": { "version": "3.0.4", - "from": "minimatch@>=2.0.0 <3.0.0||>=3.0.0 <4.0.0", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz" + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha1-UWbihkV/AzBgZL5Ul+jbsMPTIIM=", + "requires": { + "brace-expansion": "^1.1.7" + } }, "minimist": { "version": "0.0.8", - "from": "minimist@0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz" + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" + }, + "mkdirp": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "dev": true, + "requires": { + "minimist": "0.0.8" + } }, "mocha": { "version": "5.2.0", - "from": "mocha@5.2.0", "resolved": "https://registry.npmjs.org/mocha/-/mocha-5.2.0.tgz", + "integrity": "sha1-bYrlCPWRZ/lA8rWzxKYSrlDJCuY=", + "requires": { + "browser-stdout": "1.3.1", + "commander": "2.15.1", + "debug": "3.1.0", + "diff": "3.5.0", + "escape-string-regexp": "1.0.5", + "glob": "7.1.2", + "growl": "1.10.5", + "he": "1.1.1", + "minimatch": "3.0.4", + "mkdirp": "0.5.1", + "supports-color": "5.4.0" + }, "dependencies": { "commander": { "version": "2.15.1", - "from": "commander@2.15.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz" + "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz", + "integrity": "sha1-30boZ9D8Kuxmo0ZitAapzK//Ww8=" }, "debug": { "version": "3.1.0", - "from": "debug@3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz" + "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", + "integrity": "sha1-W7WgZyYotkFJVmuhaBnmFRjGcmE=", + "requires": { + "ms": "2.0.0" + } }, "diff": { "version": "3.5.0", - "from": "diff@3.5.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz" + "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", + "integrity": "sha1-gAwN0eCov7yVg1wgKtIg/jF+WhI=" }, "glob": { "version": "7.1.2", - "from": "glob@7.1.2", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz" + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", + "integrity": "sha1-wZyd+aAocC1nhhI4SmVSQExjbRU=", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } }, "growl": { "version": "1.10.5", - "from": "growl@1.10.5", - "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz" + "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", + "integrity": "sha1-8nNdwig2dPpnR4sQGBBZNVw2nl4=" }, "mkdirp": { "version": "0.5.1", - "from": "mkdirp@0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz" + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "requires": { + "minimist": "0.0.8" + } }, "supports-color": { "version": "5.4.0", - "from": "supports-color@5.4.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz" + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz", + "integrity": "sha1-HGszdALCE3YF7+GfEP7DkPb6q1Q=", + "requires": { + "has-flag": "^3.0.0" + } } } }, "module-details-from-path": { "version": "1.0.3", - "from": "module-details-from-path@>=1.0.3 <2.0.0", - "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz" + "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz", + "integrity": "sha1-EUyUlnPiqKNenTV4hSeqN7Z52is=" }, "moment": { "version": "2.24.0", - "from": "moment@>=2.10.6 <3.0.0", "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", + "integrity": "sha1-DQVdU/UFKqZTyfbraLtdEr9cK1s=", "optional": true }, "ms": { "version": "2.0.0", - "from": "ms@2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz" + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "mute-stream": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", + "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", + "dev": true }, "mv": { "version": "2.1.1", - "from": "mv@>=2.0.0 <3.0.0", "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz", + "integrity": "sha1-rmzg1vbV4KT32JN5jQPB6pVZtqI=", "optional": true, + "requires": { + "mkdirp": "~0.5.1", + "ncp": "~2.0.0", + "rimraf": "~2.4.0" + }, "dependencies": { "mkdirp": { "version": "0.5.1", - "from": "mkdirp@>=0.5.1 <0.6.0", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "optional": true + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "optional": true, + "requires": { + "minimist": "0.0.8" + } }, "rimraf": { "version": "2.4.5", - "from": "rimraf@>=2.4.0 <2.5.0", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz", - "optional": true + "integrity": "sha1-7nEM5dk6j9uFb7Xqj/Di11k0sto=", + "optional": true, + "requires": { + "glob": "^6.0.1" + } } } }, "nan": { "version": "2.11.0", - "from": "nan@>=2.0.8 <3.0.0", "resolved": "https://registry.npmjs.org/nan/-/nan-2.11.0.tgz", + "integrity": "sha1-V042Dk2VSrFpZuwQLAwEn9lhoJk=", "optional": true }, + "natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", + "dev": true + }, "ncp": { "version": "2.0.0", - "from": "ncp@>=2.0.0 <2.1.0", "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", + "integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=", "optional": true }, "negotiator": { "version": "0.6.1", - "from": "negotiator@0.6.1", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz" + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", + "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=" + }, + "nice-try": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", + "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", + "dev": true }, "nise": { "version": "1.5.0", - "from": "nise@>=1.4.6 <2.0.0", "resolved": "https://registry.npmjs.org/nise/-/nise-1.5.0.tgz", + "integrity": "sha1-0D6g5sG3XGOAFao1he3cEylJpQ0=", "dev": true, + "requires": { + "@sinonjs/formatio": "^3.1.0", + "@sinonjs/text-encoding": "^0.7.1", + "just-extend": "^4.0.2", + "lolex": "^4.1.0", + "path-to-regexp": "^1.7.0" + }, "dependencies": { "isarray": { "version": "0.0.1", - "from": "isarray@0.0.1", "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=", "dev": true }, "lolex": { "version": "4.1.0", - "from": "lolex@>=4.1.0 <5.0.0", "resolved": "https://registry.npmjs.org/lolex/-/lolex-4.1.0.tgz", + "integrity": "sha1-7N17hlOTkdgjeUejQZqorJdfD+E=", "dev": true }, "path-to-regexp": { "version": "1.7.0", - "from": "path-to-regexp@>=1.7.0 <2.0.0", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.7.0.tgz", - "dev": true + "integrity": "sha1-Wf3g9DW62suhA6hOnTvGTpa5k30=", + "dev": true, + "requires": { + "isarray": "0.0.1" + } } } }, "node-fetch": { "version": "2.6.0", - "from": "node-fetch@>=2.3.0 <3.0.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz" + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", + "integrity": "sha1-5jNFY4bUqlWGP2dqerDaqP3ssP0=" }, "node-forge": { "version": "0.8.4", - "from": "node-forge@>=0.8.0 <0.9.0", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.8.4.tgz" + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.8.4.tgz", + "integrity": "sha1-1nOGYrZhvhnicR7wGqOxghLxMDA=" }, "node-transloadit": { "version": "0.0.4", - "from": "node-transloadit@0.0.4", "resolved": "https://registry.npmjs.org/node-transloadit/-/node-transloadit-0.0.4.tgz", + "integrity": "sha1-4ZoHheON94NblO2AANHjXmg7zsE=", + "requires": { + "request": "~2.16.6", + "underscore": "1.2.1" + }, "dependencies": { "json-stringify-safe": { "version": "3.0.0", - "from": "json-stringify-safe@>=3.0.0 <3.1.0", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-3.0.0.tgz" + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-3.0.0.tgz", + "integrity": "sha1-nbew5TDH8onF6MhDKvGRwv91pbM=" }, "mime": { "version": "1.2.11", - "from": "mime@>=1.2.7 <1.3.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz" + "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz", + "integrity": "sha1-WCA+7Ybjpe8XrtK32evUfwpg3RA=" }, "qs": { "version": "0.5.6", - "from": "qs@>=0.5.4 <0.6.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-0.5.6.tgz" + "resolved": "https://registry.npmjs.org/qs/-/qs-0.5.6.tgz", + "integrity": "sha1-MbGtBYVnZRxSaSFQa5qHk5EaA4Q=" }, "request": { "version": "2.16.6", - "from": "request@>=2.16.6 <2.17.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.16.6.tgz" + "resolved": "https://registry.npmjs.org/request/-/request-2.16.6.tgz", + "integrity": "sha1-hy/kRa5y3iZrN4edatfclI+gHK0=", + "requires": { + "aws-sign": "~0.2.0", + "cookie-jar": "~0.2.0", + "forever-agent": "~0.2.0", + "form-data": "~0.0.3", + "hawk": "~0.10.2", + "json-stringify-safe": "~3.0.0", + "mime": "~1.2.7", + "node-uuid": "~1.4.0", + "oauth-sign": "~0.2.0", + "qs": "~0.5.4", + "tunnel-agent": "~0.2.0" + } }, "underscore": { "version": "1.2.1", - "from": "underscore@1.2.1", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.2.1.tgz" + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.2.1.tgz", + "integrity": "sha1-/FxrB2VnPZKi1KyLTcCqiHAuK9Q=" } } }, "node-uuid": { "version": "1.4.8", - "from": "node-uuid@>=1.4.1 <1.5.0", - "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.8.tgz" + "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.8.tgz", + "integrity": "sha1-sEDrCSOWivq/jTL7HxfxFn/auQc=" + }, + "normalize-package-data": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", + "integrity": "sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==", + "dev": true, + "requires": { + "hosted-git-info": "^2.1.4", + "resolve": "^1.10.0", + "semver": "2 || 3 || 4 || 5", + "validate-npm-package-license": "^3.0.1" + } }, "oauth-sign": { "version": "0.2.0", - "from": "oauth-sign@>=0.2.0 <0.3.0", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.2.0.tgz" + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.2.0.tgz", + "integrity": "sha1-oOahcV2u0GLzIrYit/5a/RA1tuI=" + }, + "object-inspect": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.7.0.tgz", + "integrity": "sha512-a7pEHdh1xKIAgTySUGgLMx/xwDZskN1Ud6egYYN3EdRW4ZMPNEDUTF+hwy2LUC+Bl+SyLXANnwz/jyh/qutKUw==", + "dev": true + }, + "object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true + }, + "object.assign": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.0.tgz", + "integrity": "sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==", + "dev": true, + "requires": { + "define-properties": "^1.1.2", + "function-bind": "^1.1.1", + "has-symbols": "^1.0.0", + "object-keys": "^1.0.11" + } + }, + "object.values": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.1.tgz", + "integrity": "sha512-WTa54g2K8iu0kmS/us18jEmdv1a4Wi//BZ/DTVYEcH0XhLM5NYdpDHja3gt57VrZLcNAO2WGA+KpWsDBaHt6eA==", + "dev": true, + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.1", + "function-bind": "^1.1.1", + "has": "^1.0.3" + } }, "on-finished": { "version": "2.3.0", - "from": "on-finished@>=2.3.0 <2.4.0", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz" + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", + "requires": { + "ee-first": "1.1.1" + } }, "once": { "version": "1.4.0", - "from": "once@>=1.3.0 <2.0.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz" + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "requires": { + "wrappy": "1" + } + }, + "onetime": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.0.tgz", + "integrity": "sha512-5NcSkPHhwTVFIQN+TUqXoS5+dlElHXdpAWu9I0HP20YOtIi+aZ0Ct82jdlILDxjLEAWwvm+qj1m6aEtsDVmm6Q==", + "dev": true, + "requires": { + "mimic-fn": "^2.1.0" + } + }, + "optionator": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", + "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", + "dev": true, + "requires": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.6", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "word-wrap": "~1.2.3" + } + }, + "os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", + "dev": true }, "p-limit": { "version": "2.2.0", - "from": "p-limit@>=2.2.0 <3.0.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.0.tgz" + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.0.tgz", + "integrity": "sha1-QXyZQeYCepq8ulCS3SkE4lW1+8I=", + "requires": { + "p-try": "^2.0.0" + } + }, + "p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", + "dev": true, + "requires": { + "p-limit": "^1.1.0" + } }, "p-try": { "version": "2.2.0", - "from": "p-try@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz" + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha1-yyhoVA4xPWHeWPr741zpAE1VQOY=" + }, + "parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "requires": { + "callsites": "^3.0.0" + } }, "parse-duration": { "version": "0.1.1", - "from": "parse-duration@>=0.1.1 <0.2.0", - "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-0.1.1.tgz" + "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-0.1.1.tgz", + "integrity": "sha1-ExFN3JiRwezSgANiRFVN5DZHoiY=" + }, + "parse-json": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", + "integrity": "sha1-9ID0BDTvgHQfhGkJn43qGPVaTck=", + "dev": true, + "requires": { + "error-ex": "^1.2.0" + } }, "parse-ms": { "version": "2.1.0", - "from": "parse-ms@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz" + "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz", + "integrity": "sha1-NIVlp1PUOR+lJAKZVrFyy3dTCX0=" }, "parseurl": { "version": "1.3.2", - "from": "parseurl@>=1.3.2 <1.4.0", - "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz" + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz", + "integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M=" + }, + "path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true }, "path-is-absolute": { "version": "1.0.1", - "from": "path-is-absolute@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz" + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" + }, + "path-is-inside": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", + "integrity": "sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM=", + "dev": true + }, + "path-key": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", + "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=", + "dev": true }, "path-parse": { "version": "1.0.6", - "from": "path-parse@>=1.0.5 <2.0.0", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz" + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", + "integrity": "sha1-1i27VnlAXXLEc37FhgDp3c8G0kw=" }, "path-to-regexp": { "version": "0.1.7", - "from": "path-to-regexp@0.1.7", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz" + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + }, + "path-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-2.0.0.tgz", + "integrity": "sha1-8BLMuEFbcJb8LaoQVMPXI4lZTHM=", + "dev": true, + "requires": { + "pify": "^2.0.0" + } }, "pathval": { "version": "1.1.0", - "from": "pathval@>=1.1.0 <2.0.0", "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz", + "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=", "dev": true }, "performance-now": { "version": "2.1.0", - "from": "performance-now@>=2.1.0 <3.0.0", - "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz" + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" }, "pify": { "version": "4.0.1", - "from": "pify@>=4.0.0 <5.0.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz" + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha1-SyzSXFDVmHNcUCkiJP2MbfQeMjE=" + }, + "pkg-dir": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz", + "integrity": "sha1-9tXREJ4Z1j7fQo4L1X4Sd3YVM0s=", + "dev": true, + "requires": { + "find-up": "^2.1.0" + } }, "pngcrush": { "version": "0.0.3", - "from": "pngcrush@0.0.3", - "resolved": "https://registry.npmjs.org/pngcrush/-/pngcrush-0.0.3.tgz" + "resolved": "https://registry.npmjs.org/pngcrush/-/pngcrush-0.0.3.tgz", + "integrity": "sha1-v2dW6s2h+rNJwHdo6AXMEA0o+Tc=", + "requires": { + "gettemporaryfilepath": "=0.0.1" + } + }, + "prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", + "dev": true + }, + "prettier": { + "version": "1.19.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.19.1.tgz", + "integrity": "sha512-s7PoyDv/II1ObgQunCbB9PdLmUcBZcnWOcxDh7O0N/UwDEsHyqkW+Qh28jW+mVuCdx7gLB0BotYI1Y6uI9iyew==", + "dev": true + }, + "prettier-eslint": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/prettier-eslint/-/prettier-eslint-9.0.1.tgz", + "integrity": "sha512-KZT65QTosSAqBBqmrC+RpXbsMRe7Os2YSR9cAfFbDlyPAopzA/S5bioiZ3rpziNQNSJaOxmtXSx07EQ+o2Dlug==", + "dev": true, + "requires": { + "@typescript-eslint/parser": "^1.10.2", + "common-tags": "^1.4.0", + "core-js": "^3.1.4", + "dlv": "^1.1.0", + "eslint": "^5.0.0", + "indent-string": "^4.0.0", + "lodash.merge": "^4.6.0", + "loglevel-colored-level-prefix": "^1.0.0", + "prettier": "^1.7.0", + "pretty-format": "^23.0.1", + "require-relative": "^0.8.7", + "typescript": "^3.2.1", + "vue-eslint-parser": "^2.0.2" + }, + "dependencies": { + "ansi-escapes": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz", + "integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==", + "dev": true + }, + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "dev": true + }, + "cli-cursor": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", + "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=", + "dev": true, + "requires": { + "restore-cursor": "^2.0.0" + } + }, + "eslint": { + "version": "5.16.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-5.16.0.tgz", + "integrity": "sha512-S3Rz11i7c8AA5JPv7xAH+dOyq/Cu/VXHiHXBPOU1k/JAM5dXqQPt3qcrhpHSorXmrpu2g0gkIBVXAqCpzfoZIg==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "ajv": "^6.9.1", + "chalk": "^2.1.0", + "cross-spawn": "^6.0.5", + "debug": "^4.0.1", + "doctrine": "^3.0.0", + "eslint-scope": "^4.0.3", + "eslint-utils": "^1.3.1", + "eslint-visitor-keys": "^1.0.0", + "espree": "^5.0.1", + "esquery": "^1.0.1", + "esutils": "^2.0.2", + "file-entry-cache": "^5.0.1", + "functional-red-black-tree": "^1.0.1", + "glob": "^7.1.2", + "globals": "^11.7.0", + "ignore": "^4.0.6", + "import-fresh": "^3.0.0", + "imurmurhash": "^0.1.4", + "inquirer": "^6.2.2", + "js-yaml": "^3.13.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.3.0", + "lodash": "^4.17.11", + "minimatch": "^3.0.4", + "mkdirp": "^0.5.1", + "natural-compare": "^1.4.0", + "optionator": "^0.8.2", + "path-is-inside": "^1.0.2", + "progress": "^2.0.0", + "regexpp": "^2.0.1", + "semver": "^5.5.1", + "strip-ansi": "^4.0.0", + "strip-json-comments": "^2.0.1", + "table": "^5.2.3", + "text-table": "^0.2.0" + } + }, + "eslint-scope": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", + "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==", + "dev": true, + "requires": { + "esrecurse": "^4.1.0", + "estraverse": "^4.1.1" + } + }, + "espree": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-5.0.1.tgz", + "integrity": "sha512-qWAZcWh4XE/RwzLJejfcofscgMc9CamR6Tn1+XRXNzrvUSSbiAjGOI/fggztjIi7y9VLPqnICMIPiGyr8JaZ0A==", + "dev": true, + "requires": { + "acorn": "^6.0.7", + "acorn-jsx": "^5.0.0", + "eslint-visitor-keys": "^1.0.0" + } + }, + "figures": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", + "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=", + "dev": true, + "requires": { + "escape-string-regexp": "^1.0.5" + } + }, + "globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "dev": true + }, + "inquirer": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.5.2.tgz", + "integrity": "sha512-cntlB5ghuB0iuO65Ovoi8ogLHiWGs/5yNrtUcKjFhSSiVeAIVpD7koaSU9RM8mpXw5YDi9RdYXGQMaOURB7ycQ==", + "dev": true, + "requires": { + "ansi-escapes": "^3.2.0", + "chalk": "^2.4.2", + "cli-cursor": "^2.1.0", + "cli-width": "^2.0.0", + "external-editor": "^3.0.3", + "figures": "^2.0.0", + "lodash": "^4.17.12", + "mute-stream": "0.0.7", + "run-async": "^2.2.0", + "rxjs": "^6.4.0", + "string-width": "^2.1.0", + "strip-ansi": "^5.1.0", + "through": "^2.3.6" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "requires": { + "ansi-regex": "^4.1.0" + } + } + } + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true + }, + "mimic-fn": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", + "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", + "dev": true + }, + "mute-stream": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", + "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=", + "dev": true + }, + "onetime": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", + "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=", + "dev": true, + "requires": { + "mimic-fn": "^1.0.0" + } + }, + "restore-cursor": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", + "integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=", + "dev": true, + "requires": { + "onetime": "^2.0.0", + "signal-exit": "^3.0.2" + } + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "dev": true, + "requires": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + } + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "dev": true, + "requires": { + "ansi-regex": "^3.0.0" + } + }, + "strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", + "dev": true + } + } + }, + "prettier-eslint-cli": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/prettier-eslint-cli/-/prettier-eslint-cli-5.0.0.tgz", + "integrity": "sha512-cei9UbN1aTrz3sQs88CWpvY/10PYTevzd76zoG1tdJ164OhmNTFRKPTOZrutVvscoQWzbnLKkviS3gu5JXwvZg==", + "dev": true, + "requires": { + "arrify": "^2.0.1", + "boolify": "^1.0.0", + "camelcase-keys": "^6.0.0", + "chalk": "^2.4.2", + "common-tags": "^1.8.0", + "core-js": "^3.1.4", + "eslint": "^5.0.0", + "find-up": "^4.1.0", + "get-stdin": "^7.0.0", + "glob": "^7.1.4", + "ignore": "^5.1.2", + "lodash.memoize": "^4.1.2", + "loglevel-colored-level-prefix": "^1.0.0", + "messageformat": "^2.2.1", + "prettier-eslint": "^9.0.0", + "rxjs": "^6.5.2", + "yargs": "^13.2.4" + }, + "dependencies": { + "ansi-escapes": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz", + "integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==", + "dev": true + }, + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "dev": true + }, + "cli-cursor": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", + "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=", + "dev": true, + "requires": { + "restore-cursor": "^2.0.0" + } + }, + "eslint": { + "version": "5.16.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-5.16.0.tgz", + "integrity": "sha512-S3Rz11i7c8AA5JPv7xAH+dOyq/Cu/VXHiHXBPOU1k/JAM5dXqQPt3qcrhpHSorXmrpu2g0gkIBVXAqCpzfoZIg==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "ajv": "^6.9.1", + "chalk": "^2.1.0", + "cross-spawn": "^6.0.5", + "debug": "^4.0.1", + "doctrine": "^3.0.0", + "eslint-scope": "^4.0.3", + "eslint-utils": "^1.3.1", + "eslint-visitor-keys": "^1.0.0", + "espree": "^5.0.1", + "esquery": "^1.0.1", + "esutils": "^2.0.2", + "file-entry-cache": "^5.0.1", + "functional-red-black-tree": "^1.0.1", + "glob": "^7.1.2", + "globals": "^11.7.0", + "ignore": "^4.0.6", + "import-fresh": "^3.0.0", + "imurmurhash": "^0.1.4", + "inquirer": "^6.2.2", + "js-yaml": "^3.13.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.3.0", + "lodash": "^4.17.11", + "minimatch": "^3.0.4", + "mkdirp": "^0.5.1", + "natural-compare": "^1.4.0", + "optionator": "^0.8.2", + "path-is-inside": "^1.0.2", + "progress": "^2.0.0", + "regexpp": "^2.0.1", + "semver": "^5.5.1", + "strip-ansi": "^4.0.0", + "strip-json-comments": "^2.0.1", + "table": "^5.2.3", + "text-table": "^0.2.0" + }, + "dependencies": { + "ignore": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", + "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "dev": true + } + } + }, + "eslint-scope": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", + "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==", + "dev": true, + "requires": { + "esrecurse": "^4.1.0", + "estraverse": "^4.1.1" + } + }, + "espree": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-5.0.1.tgz", + "integrity": "sha512-qWAZcWh4XE/RwzLJejfcofscgMc9CamR6Tn1+XRXNzrvUSSbiAjGOI/fggztjIi7y9VLPqnICMIPiGyr8JaZ0A==", + "dev": true, + "requires": { + "acorn": "^6.0.7", + "acorn-jsx": "^5.0.0", + "eslint-visitor-keys": "^1.0.0" + } + }, + "figures": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", + "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=", + "dev": true, + "requires": { + "escape-string-regexp": "^1.0.5" + } + }, + "find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "requires": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, + "get-stdin": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-7.0.0.tgz", + "integrity": "sha512-zRKcywvrXlXsA0v0i9Io4KDRaAw7+a1ZpjRwl9Wox8PFlVCCHra7E9c4kqXCoCM9nR5tBkaTTZRBoCm60bFqTQ==", + "dev": true + }, + "globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "dev": true + }, + "ignore": { + "version": "5.1.4", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.4.tgz", + "integrity": "sha512-MzbUSahkTW1u7JpKKjY7LCARd1fU5W2rLdxlM4kdkayuCwZImjkpluF9CM1aLewYJguPDqewLam18Y6AU69A8A==", + "dev": true + }, + "inquirer": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.5.2.tgz", + "integrity": "sha512-cntlB5ghuB0iuO65Ovoi8ogLHiWGs/5yNrtUcKjFhSSiVeAIVpD7koaSU9RM8mpXw5YDi9RdYXGQMaOURB7ycQ==", + "dev": true, + "requires": { + "ansi-escapes": "^3.2.0", + "chalk": "^2.4.2", + "cli-cursor": "^2.1.0", + "cli-width": "^2.0.0", + "external-editor": "^3.0.3", + "figures": "^2.0.0", + "lodash": "^4.17.12", + "mute-stream": "0.0.7", + "run-async": "^2.2.0", + "rxjs": "^6.4.0", + "string-width": "^2.1.0", + "strip-ansi": "^5.1.0", + "through": "^2.3.6" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "requires": { + "ansi-regex": "^4.1.0" + } + } + } + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true + }, + "locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "requires": { + "p-locate": "^4.1.0" + } + }, + "mimic-fn": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", + "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", + "dev": true + }, + "mute-stream": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", + "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=", + "dev": true + }, + "onetime": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", + "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=", + "dev": true, + "requires": { + "mimic-fn": "^1.0.0" + } + }, + "p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "requires": { + "p-limit": "^2.2.0" + } + }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true + }, + "restore-cursor": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", + "integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=", + "dev": true, + "requires": { + "onetime": "^2.0.0", + "signal-exit": "^3.0.2" + } + }, + "string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "dev": true, + "requires": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + } + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "dev": true, + "requires": { + "ansi-regex": "^3.0.0" + } + }, + "strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", + "dev": true + } + } + }, + "prettier-linter-helpers": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz", + "integrity": "sha512-GbK2cP9nraSSUF9N2XwUwqfzlAFlMNYYl+ShE/V+H8a9uNl/oUqB1w2EL54Jh0OlyRSd8RfWYJ3coVS4TROP2w==", + "dev": true, + "requires": { + "fast-diff": "^1.1.2" + } + }, + "pretty-format": { + "version": "23.6.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-23.6.0.tgz", + "integrity": "sha512-zf9NV1NSlDLDjycnwm6hpFATCGl/K1lt0R/GdkAK2O5LN/rwJoB+Mh93gGJjut4YbmecbfgLWVGSTCr0Ewvvbw==", + "dev": true, + "requires": { + "ansi-regex": "^3.0.0", + "ansi-styles": "^3.2.0" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "dev": true + } + } }, "pretty-ms": { "version": "4.0.0", - "from": "pretty-ms@>=4.0.0 <5.0.0", - "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-4.0.0.tgz" + "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-4.0.0.tgz", + "integrity": "sha1-Mbr0G5T9AiJwmKqgO9YmCOsNbpI=", + "requires": { + "parse-ms": "^2.0.0" + } }, "process-nextick-args": { "version": "2.0.0", - "from": "process-nextick-args@>=2.0.0 <2.1.0", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz" + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", + "integrity": "sha1-o31zL0JxtKsa0HDTVQjoKQeI/6o=" + }, + "progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "dev": true }, "prom-client": { "version": "11.5.1", - "from": "prom-client@>=11.1.3 <12.0.0", - "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-11.5.1.tgz" + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-11.5.1.tgz", + "integrity": "sha1-FcZsrN7EUwELz68EEJvMNOa92pw=", + "requires": { + "tdigest": "^0.1.1" + } }, "protobufjs": { "version": "6.8.8", - "from": "protobufjs@>=6.8.6 <6.9.0", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.8.tgz", + "integrity": "sha1-yLTxKC/XqQ5vWxCe0RyEr4KQjnw=", + "requires": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/long": "^4.0.0", + "@types/node": "^10.1.0", + "long": "^4.0.0" + }, "dependencies": { "@types/node": { "version": "10.14.9", - "from": "@types/node@>=10.1.0 <11.0.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.9.tgz" + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.9.tgz", + "integrity": "sha1-Lo1ngDnSeUPOU6GRM4YTMif9kGY=" } } }, "proxy-addr": { "version": "2.0.4", - "from": "proxy-addr@>=2.0.3 <2.1.0", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.4.tgz" + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.4.tgz", + "integrity": "sha1-7PxzO/Iv+Mb0B/onUye5q2fki5M=", + "requires": { + "forwarded": "~0.1.2", + "ipaddr.js": "1.8.0" + } }, "psl": { "version": "1.1.32", - "from": "psl@>=1.1.24 <2.0.0", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.32.tgz" + "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.32.tgz", + "integrity": "sha1-PxMnF88vnBaXJLK2yvNzz2lBmNs=" }, "punycode": { "version": "1.3.2", - "from": "punycode@1.3.2", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz" + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" }, "qs": { "version": "6.5.2", - "from": "qs@6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz" + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha1-yzroBuh0BERYTvFUzo7pjUA/PjY=" }, "querystring": { "version": "0.2.0", - "from": "querystring@0.2.0", - "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz" + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, + "quick-lru": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-4.0.1.tgz", + "integrity": "sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==", + "dev": true + }, + "ramda": { + "version": "0.26.1", + "resolved": "https://registry.npmjs.org/ramda/-/ramda-0.26.1.tgz", + "integrity": "sha512-hLWjpy7EnsDBb0p+Z3B7rPi3GDeRG5ZtiI33kJhTt+ORCd38AbAIjB/9zRIUoeTbE/AVX5ZkU7m6bznsvrf8eQ==", + "dev": true }, "range-parser": { "version": "1.2.0", - "from": "range-parser@>=1.0.2 <2.0.0", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz" + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz", + "integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4=" }, "raven": { "version": "1.1.3", - "from": "raven@1.1.3", "resolved": "https://registry.npmjs.org/raven/-/raven-1.1.3.tgz", + "integrity": "sha1-QnPBrm005CMPUbLAEEGjK5Iygio=", + "requires": { + "cookie": "0.3.1", + "json-stringify-safe": "5.0.1", + "lsmod": "1.0.0", + "stack-trace": "0.0.9", + "uuid": "3.0.0" + }, "dependencies": { "uuid": { "version": "3.0.0", - "from": "uuid@3.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.0.0.tgz" + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.0.0.tgz", + "integrity": "sha1-Zyj8BFnEUNeWqZwxg3VpvfZy1yg=" } } }, "raw-body": { "version": "2.3.3", - "from": "raw-body@2.3.3", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.3.tgz" + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.3.tgz", + "integrity": "sha1-GzJOzmtXBuFThVvBFIxlu39uoMM=", + "requires": { + "bytes": "3.0.0", + "http-errors": "1.6.3", + "iconv-lite": "0.4.23", + "unpipe": "1.0.0" + } + }, + "read-pkg": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-2.0.0.tgz", + "integrity": "sha1-jvHAYjxqbbDcZxPEv6xGMysjaPg=", + "dev": true, + "requires": { + "load-json-file": "^2.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^2.0.0" + } + }, + "read-pkg-up": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-2.0.0.tgz", + "integrity": "sha1-a3KoBImE4MQeeVEP1en6mbO1Sb4=", + "dev": true, + "requires": { + "find-up": "^2.0.0", + "read-pkg": "^2.0.0" + } }, "readable-stream": { "version": "1.0.34", - "from": "readable-stream@>=1.0.2 <1.1.0", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + }, "dependencies": { "isarray": { "version": "0.0.1", - "from": "isarray@0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" } } }, "recluster": { "version": "0.3.7", - "from": "recluster@>=0.3.7 <0.4.0", - "resolved": "https://registry.npmjs.org/recluster/-/recluster-0.3.7.tgz" + "resolved": "https://registry.npmjs.org/recluster/-/recluster-0.3.7.tgz", + "integrity": "sha1-aKRx3ZC2obl3ZjTPdpZAWutWeJU=" + }, + "regexpp": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-2.0.1.tgz", + "integrity": "sha512-lv0M6+TkDVniA3aD1Eg0DVpfU/booSu7Eev3TDO/mZKHBfVjgCGTV4t4buppESEYDtkArYFOxTJWv6S5C+iaNw==", + "dev": true }, "request": { "version": "2.14.0", - "from": "request@2.14.0", "resolved": "https://registry.npmjs.org/request/-/request-2.14.0.tgz", + "integrity": "sha1-DYrLsLFMGrguAAt9OB+oyA0afYg=", + "requires": { + "form-data": "~0.0.3", + "mime": "~1.2.7" + }, "dependencies": { "form-data": { "version": "0.0.7", - "from": "form-data@~0.0.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-0.0.7.tgz", + "integrity": "sha1-chEYKiaiZs45cQ3IvEqBtwQIWb4=", + "requires": { + "async": "~0.1.9", + "combined-stream": "~0.0.4", + "mime": "~1.2.2" + }, "dependencies": { "async": { "version": "0.1.22", - "from": "async@~0.1.9" + "resolved": "https://registry.npmjs.org/async/-/async-0.1.22.tgz", + "integrity": "sha1-D8GqoIig4+8Ovi2IMbqw3PiEUGE=" }, "combined-stream": { "version": "0.0.4", - "from": "combined-stream@~0.0.4", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.4.tgz", + "integrity": "sha1-LRpDNH2+lRWkonlnMuW4hHOECyI=", + "requires": { + "delayed-stream": "0.0.5" + }, "dependencies": { "delayed-stream": { "version": "0.0.5", - "from": "delayed-stream@0.0.5" + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz", + "integrity": "sha1-1LH0OpPoKW3+AmlPRoC8N6MTxz8=" } } } @@ -1583,363 +4250,1013 @@ }, "mime": { "version": "1.2.9", - "from": "mime@~1.2.7" + "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.9.tgz", + "integrity": "sha1-AJzUCGe9Nd5SGzuWbwTi+NTRPQk=" } } }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", + "dev": true + }, "require-in-the-middle": { "version": "4.0.0", - "from": "require-in-the-middle@>=4.0.0 <5.0.0", "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-4.0.0.tgz", + "integrity": "sha1-PHUoik7EgM30S8d950T4q+WFQFs=", + "requires": { + "debug": "^4.1.1", + "module-details-from-path": "^1.0.3", + "resolve": "^1.10.0" + }, "dependencies": { "debug": { "version": "4.1.1", - "from": "debug@>=4.1.1 <5.0.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz" + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha1-O3ImAlUQnGtYnO4FDx1RYTlmR5E=", + "requires": { + "ms": "^2.1.1" + } }, "ms": { "version": "2.1.2", - "from": "ms@^2.1.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha1-0J0fNXtEP0kzgqjrPM0YOHKuYAk=" } } }, "require-like": { "version": "0.1.2", - "from": "require-like@0.1.2", "resolved": "https://registry.npmjs.org/require-like/-/require-like-0.1.2.tgz", + "integrity": "sha1-rW8wwTvs15cBDEaK+ndcDAprR/o=", + "dev": true + }, + "require-main-filename": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", + "dev": true + }, + "require-relative": { + "version": "0.8.7", + "resolved": "https://registry.npmjs.org/require-relative/-/require-relative-0.8.7.tgz", + "integrity": "sha1-eZlTn8ngR6N5KPoZb44VY9q9Nt4=", "dev": true }, "resolve": { "version": "1.11.0", - "from": "resolve@>=1.10.0 <2.0.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.0.tgz" + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.0.tgz", + "integrity": "sha1-QBSHC6KWF2uGND1Qtg87UGCc4jI=", + "requires": { + "path-parse": "^1.0.6" + } + }, + "resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true }, "response": { "version": "0.14.0", - "from": "response@0.14.0", "resolved": "https://registry.npmjs.org/response/-/response-0.14.0.tgz", + "integrity": "sha1-BmNS/z5rAm0EdYCUB2Y7Rob9JpY=", + "requires": { + "best-encoding": "^0.1.1", + "bl": "~0.7.0", + "caseless": "^0.3.0", + "mime": "~1.2.11" + }, "dependencies": { "mime": { "version": "1.2.11", - "from": "mime@>=1.2.11 <1.3.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz" + "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz", + "integrity": "sha1-WCA+7Ybjpe8XrtK32evUfwpg3RA=" } } }, + "restore-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", + "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", + "dev": true, + "requires": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + } + }, "retry-axios": { "version": "0.3.2", - "from": "retry-axios@0.3.2", - "resolved": "https://registry.npmjs.org/retry-axios/-/retry-axios-0.3.2.tgz" + "resolved": "https://registry.npmjs.org/retry-axios/-/retry-axios-0.3.2.tgz", + "integrity": "sha1-V1fID1hbTMTEmGqi/9R6YMbTXhM=" }, "retry-request": { "version": "4.0.0", - "from": "retry-request@>=4.0.0 <5.0.0", - "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-4.0.0.tgz" + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-4.0.0.tgz", + "integrity": "sha1-XDZhZiebPhDp16oTJ0RnoFy2kpA=", + "requires": { + "through2": "^2.0.0" + } }, "rimraf": { "version": "2.2.8", - "from": "rimraf@2.2.8", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.2.8.tgz" + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.2.8.tgz", + "integrity": "sha1-5Dm+Kq7jJzIZUnMPmaiSnk/FBYI=" + }, + "run-async": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.3.0.tgz", + "integrity": "sha1-A3GrSuC91yDUFm19/aZP96RFpsA=", + "dev": true, + "requires": { + "is-promise": "^2.1.0" + } + }, + "rxjs": { + "version": "6.5.3", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.3.tgz", + "integrity": "sha512-wuYsAYYFdWTAnAaPoKGNhfpWwKZbJW+HgAJ+mImp+Epl7BG8oNWBCTyRM8gba9k4lk8BgWdoYm21Mo/RYhhbgA==", + "dev": true, + "requires": { + "tslib": "^1.9.0" + } }, "safe-buffer": { "version": "5.1.1", - "from": "safe-buffer@5.1.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz" + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", + "integrity": "sha1-iTMSr2myEj3vcfV4iQAWce6yyFM=" }, "safe-json-stringify": { "version": "1.2.0", - "from": "safe-json-stringify@>=1.0.0 <2.0.0", "resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.2.0.tgz", + "integrity": "sha1-NW5EvJjx+TzkXfFLzXwBzahuCv0=", "optional": true }, "safer-buffer": { "version": "2.1.2", - "from": "safer-buffer@>=2.1.2 <3.0.0", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz" + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha1-RPoWGwGHuVSd2Eu5GAL5vYOFzWo=" }, "sandboxed-module": { "version": "2.0.3", - "from": "sandboxed-module@latest", "resolved": "https://registry.npmjs.org/sandboxed-module/-/sandboxed-module-2.0.3.tgz", - "dev": true + "integrity": "sha1-x+VFkzm7y6KMUwPusz9ug4e/upY=", + "dev": true, + "requires": { + "require-like": "0.1.2", + "stack-trace": "0.0.9" + } }, "sax": { "version": "1.2.1", - "from": "sax@1.2.1", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz" + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" }, "semver": { "version": "6.1.1", - "from": "semver@>=6.0.0 <7.0.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.1.1.tgz" + "resolved": "https://registry.npmjs.org/semver/-/semver-6.1.1.tgz", + "integrity": "sha1-U/U9qbMLIQPNTxXqs6GOy8shDJs=" }, "send": { "version": "0.16.2", - "from": "send@0.16.2", "resolved": "https://registry.npmjs.org/send/-/send-0.16.2.tgz", + "integrity": "sha1-bsyh4PjBVtFBWXVZhI32RzCmu8E=", + "requires": { + "debug": "2.6.9", + "depd": "~1.1.2", + "destroy": "~1.0.4", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "~1.6.2", + "mime": "1.4.1", + "ms": "2.0.0", + "on-finished": "~2.3.0", + "range-parser": "~1.2.0", + "statuses": "~1.4.0" + }, "dependencies": { "statuses": { "version": "1.4.0", - "from": "statuses@>=1.4.0 <1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz" + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz", + "integrity": "sha1-u3PURtonlhBu/MG2AaJT1sRr0Ic=" } } }, "serve-static": { "version": "1.13.2", - "from": "serve-static@1.13.2", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.13.2.tgz" + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.13.2.tgz", + "integrity": "sha1-CV6Ecv1bRiN9tQzkhqQ/S4bGzsE=", + "requires": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.2", + "send": "0.16.2" + } + }, + "set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", + "dev": true }, "setprototypeof": { "version": "1.1.0", - "from": "setprototypeof@1.1.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz" + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", + "integrity": "sha1-0L2FU2iHtv58DYGMuWLZ2RxU5lY=" }, "settings-sharelatex": { "version": "1.1.0", - "from": "settings-sharelatex@1.1.0", "resolved": "https://registry.npmjs.org/settings-sharelatex/-/settings-sharelatex-1.1.0.tgz", + "integrity": "sha1-Tv4vUpPbjxwVlnEEx5BfqHD/mS0=", + "requires": { + "coffee-script": "1.6.0" + }, "dependencies": { "coffee-script": { "version": "1.6.0", - "from": "coffee-script@1.6.0", - "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz" + "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz", + "integrity": "sha1-gIs5bhEPU9AhoZpO8fZb4OjjX6M=" } } }, + "shebang-command": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", + "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", + "dev": true, + "requires": { + "shebang-regex": "^1.0.0" + } + }, + "shebang-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", + "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=", + "dev": true + }, "shimmer": { "version": "1.2.1", - "from": "shimmer@>=1.2.0 <2.0.0", - "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz" + "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", + "integrity": "sha1-YQhZ994ye1h+/r9QH7QxF/mv8zc=" + }, + "signal-exit": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", + "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", + "dev": true }, "sinon": { "version": "7.1.1", - "from": "sinon@7.1.1", "resolved": "https://registry.npmjs.org/sinon/-/sinon-7.1.1.tgz", - "dev": true + "integrity": "sha1-EgLzF6oU2Ty5tp/1C2vUnA4F/8k=", + "dev": true, + "requires": { + "@sinonjs/commons": "^1.2.0", + "@sinonjs/formatio": "^3.0.0", + "@sinonjs/samsam": "^2.1.2", + "diff": "^3.5.0", + "lodash.get": "^4.4.2", + "lolex": "^3.0.0", + "nise": "^1.4.6", + "supports-color": "^5.5.0", + "type-detect": "^4.0.8" + } + }, + "slice-ansi": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz", + "integrity": "sha512-Qu+VC3EwYLldKa1fCxuuvULvSJOKEgk9pi8dZeCVK7TqBfUNTH4sFkk4joj8afVSfAYgJoSOetjx9QWOJ5mYoQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.0", + "astral-regex": "^1.0.0", + "is-fullwidth-code-point": "^2.0.0" + }, + "dependencies": { + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true + } + } }, "sntp": { "version": "0.1.4", - "from": "sntp@>=0.1.0 <0.2.0", - "resolved": "https://registry.npmjs.org/sntp/-/sntp-0.1.4.tgz" + "resolved": "https://registry.npmjs.org/sntp/-/sntp-0.1.4.tgz", + "integrity": "sha1-XvSBuVGnspr/30r9fyaDj8ESD4Q=", + "requires": { + "hoek": "0.7.x" + } }, "source-map": { "version": "0.6.1", - "from": "source-map@>=0.6.1 <0.7.0", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz" + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha1-dHIq8y6WFOnCh6jQu95IteLxomM=" + }, + "spdx-correct": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.0.tgz", + "integrity": "sha512-lr2EZCctC2BNR7j7WzJ2FpDznxky1sjfxvvYEyzxNyb6lZXHODmEoJeFu4JupYlkfha1KZpJyoqiJ7pgA1qq8Q==", + "dev": true, + "requires": { + "spdx-expression-parse": "^3.0.0", + "spdx-license-ids": "^3.0.0" + } + }, + "spdx-exceptions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.2.0.tgz", + "integrity": "sha512-2XQACfElKi9SlVb1CYadKDXvoajPgBVPn/gOQLrTvHdElaVhr7ZEbqJaRnJLVNeaI4cMEAgVCeBMKF6MWRDCRA==", + "dev": true + }, + "spdx-expression-parse": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz", + "integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==", + "dev": true, + "requires": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "spdx-license-ids": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.5.tgz", + "integrity": "sha512-J+FWzZoynJEXGphVIS+XEh3kFSjZX/1i9gFBaWQcB+/tmpe2qUsSBABpcxqxnAxFdiUFEgAX1bjYGQvIZmoz9Q==", + "dev": true }, "split": { "version": "1.0.1", - "from": "split@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz" + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha1-YFvZvjA6pZ+zX5Ip++oN3snqB9k=", + "requires": { + "through": "2" + } + }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "dev": true }, "sshpk": { "version": "1.16.1", - "from": "sshpk@>=1.7.0 <2.0.0", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz" + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", + "integrity": "sha1-+2YcC+8ps520B2nuOfpwCT1vaHc=", + "requires": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" + } }, "stack-trace": { "version": "0.0.9", - "from": "stack-trace@0.0.9", - "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.9.tgz" + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.9.tgz", + "integrity": "sha1-qPbq7KkGdMMz58Q5U/J1tFFRBpU=" }, "statsd-parser": { "version": "0.0.4", - "from": "statsd-parser@>=0.0.4 <0.1.0", - "resolved": "https://registry.npmjs.org/statsd-parser/-/statsd-parser-0.0.4.tgz" + "resolved": "https://registry.npmjs.org/statsd-parser/-/statsd-parser-0.0.4.tgz", + "integrity": "sha1-y9JDlTzELv/VSLXSI4jtaJ7GOb0=" }, "statuses": { "version": "1.5.0", - "from": "statuses@>=1.4.0 <2.0.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz" + "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" }, "stream-browserify": { "version": "2.0.1", - "from": "stream-browserify@>=2.0.1 <3.0.0", "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.1.tgz", + "integrity": "sha1-ZiZu5fm9uZQKTkUUyvtDu3Hlyds=", + "requires": { + "inherits": "~2.0.1", + "readable-stream": "^2.0.2" + }, "dependencies": { "readable-stream": { "version": "2.3.6", - "from": "readable-stream@>=2.0.2 <3.0.0", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz" + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha1-sRwn2IuP8fvgcGQ8+UsMea4bCq8=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } }, "string_decoder": { "version": "1.1.1", - "from": "string_decoder@>=1.1.1 <1.2.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz" + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha1-nPFhG6YmhdcDCunkujQUnDrwP8g=", + "requires": { + "safe-buffer": "~5.1.0" + } } } }, "stream-buffers": { "version": "0.2.6", - "from": "stream-buffers@>=0.2.5 <0.3.0", - "resolved": "https://registry.npmjs.org/stream-buffers/-/stream-buffers-0.2.6.tgz" + "resolved": "https://registry.npmjs.org/stream-buffers/-/stream-buffers-0.2.6.tgz", + "integrity": "sha1-GBwI1bs2kARfaUAbmuanoM8zE/w=" }, "stream-counter": { "version": "1.0.0", - "from": "stream-counter@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/stream-counter/-/stream-counter-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/stream-counter/-/stream-counter-1.0.0.tgz", + "integrity": "sha1-kc8lac5NxQYf6816yyY5SloRR1E=" }, "stream-shift": { "version": "1.0.0", - "from": "stream-shift@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.0.tgz", + "integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI=" + }, + "string-width": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz", + "integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==", + "dev": true, + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + }, + "dependencies": { + "strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.0" + } + } + } + }, + "string.prototype.trimleft": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/string.prototype.trimleft/-/string.prototype.trimleft-2.1.0.tgz", + "integrity": "sha512-FJ6b7EgdKxxbDxc79cOlok6Afd++TTs5szo+zJTUyow3ycrRfJVE2pq3vcN53XexvKZu/DJMDfeI/qMiZTrjTw==", + "dev": true, + "requires": { + "define-properties": "^1.1.3", + "function-bind": "^1.1.1" + } + }, + "string.prototype.trimright": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/string.prototype.trimright/-/string.prototype.trimright-2.1.0.tgz", + "integrity": "sha512-fXZTSV55dNBwv16uw+hh5jkghxSnc5oHq+5K/gXgizHwAvMetdAJlHqqoFC1FSDVPYWLkAKl2cxpUT41sV7nSg==", + "dev": true, + "requires": { + "define-properties": "^1.1.3", + "function-bind": "^1.1.1" + } }, "string_decoder": { "version": "0.10.31", - "from": "string_decoder@>=0.10.0 <0.11.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "requires": { + "ansi-regex": "^4.1.0" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true + } + } + }, + "strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=", + "dev": true + }, + "strip-json-comments": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.0.1.tgz", + "integrity": "sha512-VTyMAUfdm047mwKl+u79WIdrZxtFtn+nBxHeb844XBQ9uMNTuTHdx2hc5RiAJYqwTj3wc/xe5HLSdJSkJ+WfZw==", + "dev": true }, "supports-color": { "version": "5.5.0", - "from": "supports-color@>=5.5.0 <6.0.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "dev": true + "integrity": "sha1-4uaaRKyHcveKHsCzW2id9lMO/I8=", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + }, + "table": { + "version": "5.4.6", + "resolved": "https://registry.npmjs.org/table/-/table-5.4.6.tgz", + "integrity": "sha512-wmEc8m4fjnob4gt5riFRtTu/6+4rSe12TpAELNSqHMfF3IqnA+CH37USM6/YR3qRZv7e56kAEAtd6nKZaxe0Ug==", + "dev": true, + "requires": { + "ajv": "^6.10.2", + "lodash": "^4.17.14", + "slice-ansi": "^2.1.0", + "string-width": "^3.0.0" + }, + "dependencies": { + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + } + } }, "tdigest": { "version": "0.1.1", - "from": "tdigest@>=0.1.1 <0.2.0", - "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.1.tgz" + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.1.tgz", + "integrity": "sha1-Ljyyw56kSeVdHmzZEReszKRYgCE=", + "requires": { + "bintrees": "1.0.1" + } }, "teeny-request": { "version": "3.11.3", - "from": "teeny-request@>=3.6.0 <4.0.0", "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-3.11.3.tgz", + "integrity": "sha1-M1xin3ZF5dZZk2LfLzIwxMvCOlU=", + "requires": { + "https-proxy-agent": "^2.2.1", + "node-fetch": "^2.2.0", + "uuid": "^3.3.2" + }, "dependencies": { "uuid": { "version": "3.3.2", - "from": "uuid@^3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz" + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha1-G0r0lV6zB3xQHCOHL8ZROBFYcTE=" } } }, + "text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", + "dev": true + }, "through": { "version": "2.3.8", - "from": "through@>=2.0.0 <3.0.0", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz" + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, "through2": { "version": "2.0.5", - "from": "through2@>=2.0.3 <3.0.0", "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", + "integrity": "sha1-AcHjnrMdB8t9A6lqcIIyYLIxMs0=", + "requires": { + "readable-stream": "~2.3.6", + "xtend": "~4.0.1" + }, "dependencies": { "readable-stream": { "version": "2.3.6", - "from": "readable-stream@^2.1.5", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz" + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha1-sRwn2IuP8fvgcGQ8+UsMea4bCq8=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } }, "string_decoder": { "version": "1.1.1", - "from": "string_decoder@~1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz" + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha1-nPFhG6YmhdcDCunkujQUnDrwP8g=", + "requires": { + "safe-buffer": "~5.1.0" + } } } }, + "tmp": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", + "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "dev": true, + "requires": { + "os-tmpdir": "~1.0.2" + } + }, + "to-fast-properties": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", + "integrity": "sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4=", + "dev": true + }, "tough-cookie": { "version": "2.4.3", - "from": "tough-cookie@>=2.4.3 <2.5.0", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", + "integrity": "sha1-U/Nto/R3g7CSWvoG/587FlKA94E=", + "requires": { + "psl": "^1.1.24", + "punycode": "^1.4.1" + }, "dependencies": { "punycode": { "version": "1.4.1", - "from": "punycode@>=1.4.1 <2.0.0", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz" + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" } } }, + "tslib": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.10.0.tgz", + "integrity": "sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ==", + "dev": true + }, "tunnel-agent": { "version": "0.2.0", - "from": "tunnel-agent@>=0.2.0 <0.3.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.2.0.tgz" + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.2.0.tgz", + "integrity": "sha1-aFPCr7GyEJ5FYp5JK9419Fnqaeg=" }, "tweetnacl": { "version": "0.14.5", - "from": "tweetnacl@>=0.14.0 <0.15.0", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz" + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" + }, + "type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", + "dev": true, + "requires": { + "prelude-ls": "~1.1.2" + } }, "type-detect": { "version": "4.0.8", - "from": "type-detect@>=4.0.5 <5.0.0", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", + "integrity": "sha1-dkb7XxiHHPu3dJ5pvTmmOI63RQw=", + "dev": true + }, + "type-fest": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", + "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", "dev": true }, "type-is": { "version": "1.6.16", - "from": "type-is@>=1.6.16 <1.7.0", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz" + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz", + "integrity": "sha1-+JzjQVQcZysl7nrjxz3uOyvlAZQ=", + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.18" + } + }, + "typescript": { + "version": "3.7.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.7.3.tgz", + "integrity": "sha512-Mcr/Qk7hXqFBXMN7p7Lusj1ktCBydylfQM/FZCk5glCNQJrCUKPkMHdo9R0MTFWsC/4kPFvDS0fDPvukfCkFsw==", + "dev": true }, "underscore": { "version": "1.5.2", - "from": "underscore@>=1.5.2 <1.6.0", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.5.2.tgz" + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.5.2.tgz", + "integrity": "sha1-EzXF5PXm0zu7SwBrqMhqAPVW3gg=" }, "unpipe": { "version": "1.0.0", - "from": "unpipe@1.0.0", - "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz" + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" }, "uri-js": { "version": "4.2.2", - "from": "uri-js@>=4.2.2 <5.0.0", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", + "integrity": "sha1-lMVA4f93KVbiKZUHwBCupsiDjrA=", + "requires": { + "punycode": "^2.1.0" + }, "dependencies": { "punycode": { "version": "2.1.1", - "from": "punycode@>=2.1.0 <3.0.0", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz" + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha1-tYsBCsQMIsVldhbI0sLALHv0eew=" } } }, "url": { "version": "0.10.3", - "from": "url@0.10.3", - "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz" + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } }, "util-deprecate": { "version": "1.0.2", - "from": "util-deprecate@>=1.0.1 <1.1.0", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz" + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, "utils-merge": { "version": "1.0.1", - "from": "utils-merge@1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz" + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" + }, + "v8-compile-cache": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.0.tgz", + "integrity": "sha512-usZBT3PW+LOjM25wbqIlZwPeJV+3OSz3M1k1Ws8snlW39dZyYL9lOGC5FgPVHfk0jKmjiDV8Z0mIbVQPiwFs7g==", + "dev": true + }, + "validate-npm-package-license": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", + "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", + "dev": true, + "requires": { + "spdx-correct": "^3.0.0", + "spdx-expression-parse": "^3.0.0" + } }, "vary": { "version": "1.1.2", - "from": "vary@>=1.1.2 <1.2.0", - "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz" + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" }, "verror": { "version": "1.10.0", - "from": "verror@1.10.0", - "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz" + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "requires": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + } + }, + "vue-eslint-parser": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/vue-eslint-parser/-/vue-eslint-parser-2.0.3.tgz", + "integrity": "sha512-ZezcU71Owm84xVF6gfurBQUGg8WQ+WZGxgDEQu1IHFBZNx7BFZg3L1yHxrCBNNwbwFtE1GuvfJKMtb6Xuwc/Bw==", + "dev": true, + "requires": { + "debug": "^3.1.0", + "eslint-scope": "^3.7.1", + "eslint-visitor-keys": "^1.0.0", + "espree": "^3.5.2", + "esquery": "^1.0.0", + "lodash": "^4.17.4" + }, + "dependencies": { + "acorn-jsx": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-3.0.1.tgz", + "integrity": "sha1-r9+UiPsezvyDSPb7IvRk4ypYs2s=", + "dev": true, + "requires": { + "acorn": "^3.0.4" + } + }, + "eslint-scope": { + "version": "3.7.3", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-3.7.3.tgz", + "integrity": "sha512-W+B0SvF4gamyCTmUc+uITPY0989iXVfKvhwtmJocTaYoc/3khEHmEmvfY/Gn9HA9VV75jrQECsHizkNw1b68FA==", + "dev": true, + "requires": { + "esrecurse": "^4.1.0", + "estraverse": "^4.1.1" + } + }, + "espree": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/espree/-/espree-3.5.4.tgz", + "integrity": "sha512-yAcIQxtmMiB/jL32dzEp2enBeidsB7xWPLNiw3IIkpVds1P+h7qF9YwJq1yUNzp2OKXgAprs4F61ih66UsoD1A==", + "dev": true, + "requires": { + "acorn": "^5.5.0", + "acorn-jsx": "^3.0.0" + } + } + } + }, + "which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + }, + "which-module": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", + "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=", + "dev": true + }, + "word-wrap": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", + "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", + "dev": true + }, + "wrap-ansi": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz", + "integrity": "sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.0", + "string-width": "^3.0.0", + "strip-ansi": "^5.0.0" + }, + "dependencies": { + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + } + } }, "wrappy": { "version": "1.0.2", - "from": "wrappy@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz" + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + }, + "write": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/write/-/write-1.0.3.tgz", + "integrity": "sha512-/lg70HAjtkUgWPVZhZcm+T4hkL8Zbtp1nFNOn3lRrxnlv50SRBv7cR7RqR+GMsd3hUXy9hWBo4CHTbFTcOYwig==", + "dev": true, + "requires": { + "mkdirp": "^0.5.1" + } }, "xml2js": { "version": "0.4.19", - "from": "xml2js@0.4.19", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz" + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha1-aGwg8hMgnpSr8NG88e+qKRx4J6c=", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } }, "xmlbuilder": { "version": "9.0.7", - "from": "xmlbuilder@>=9.0.1 <9.1.0", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz" + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" }, "xtend": { "version": "4.0.1", - "from": "xtend@>=4.0.1 <4.1.0", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz" + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", + "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" + }, + "y18n": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", + "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==", + "dev": true }, "yallist": { "version": "3.0.3", - "from": "yallist@>=3.0.2 <4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz" + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", + "integrity": "sha1-tLBJ4xS+VF486AIjbWzSLNkcPek=" + }, + "yargs": { + "version": "13.3.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-13.3.0.tgz", + "integrity": "sha512-2eehun/8ALW8TLoIl7MVaRUrg+yCnenu8B4kBlRxj3GJGDKU1Og7sMXPNm1BYyM1DOJmTZ4YeN/Nwxv+8XJsUA==", + "dev": true, + "requires": { + "cliui": "^5.0.0", + "find-up": "^3.0.0", + "get-caller-file": "^2.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^3.0.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^13.1.1" + }, + "dependencies": { + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "requires": { + "locate-path": "^3.0.0" + } + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true + }, + "locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "requires": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + } + }, + "p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "requires": { + "p-limit": "^2.0.0" + } + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + } + } + }, + "yargs-parser": { + "version": "13.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.1.tgz", + "integrity": "sha512-oVAVsHz6uFrg3XQheFII8ESO2ssAf9luWuAd6Wexsu4F3OtIW0o8IribPXYrD4WC24LWtPrJlGy87y5udK+dxQ==", + "dev": true, + "requires": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + } } } } diff --git a/services/filestore/package.json b/services/filestore/package.json index bcd8011f10..708656e0d6 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -46,8 +46,22 @@ "underscore": "~1.5.2" }, "devDependencies": { + "babel-eslint": "^10.0.3", "bunyan": "^1.3.5", "chai": "4.2.0", + "eslint": "^6.7.2", + "eslint-config-prettier": "^6.7.0", + "eslint-config-standard": "^14.1.0", + "eslint-plugin-chai-expect": "^2.1.0", + "eslint-plugin-chai-friendly": "^0.5.0", + "eslint-plugin-import": "^2.19.1", + "eslint-plugin-mocha": "^6.2.2", + "eslint-plugin-node": "^10.0.0", + "eslint-plugin-prettier": "^3.1.2", + "eslint-plugin-promise": "^4.2.1", + "eslint-plugin-standard": "^4.0.1", + "prettier-eslint": "^9.0.1", + "prettier-eslint-cli": "^5.0.0", "sandboxed-module": "2.0.3", "sinon": "7.1.1" } From ae3d25c969cba090dd1f319c4f109860fd3625b8 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 16 Dec 2019 10:16:50 +0000 Subject: [PATCH 05/73] Decaffeinate: add eslint and prettier rc files --- services/filestore/.eslintrc | 67 ++++++ services/filestore/.prettierrc | 4 + services/filestore/npm-shrinkwrap.json | 292 +++++++++++++++++++++++++ 3 files changed, 363 insertions(+) create mode 100644 services/filestore/.eslintrc create mode 100644 services/filestore/.prettierrc diff --git a/services/filestore/.eslintrc b/services/filestore/.eslintrc new file mode 100644 index 0000000000..f8776c68a0 --- /dev/null +++ b/services/filestore/.eslintrc @@ -0,0 +1,67 @@ +{ + "extends": [ + "standard", + "prettier", + "prettier/standard", + ], + "plugins": [ + "mocha", + "chai-expect", + "chai-friendly" + ], + "parser": "babel-eslint", + "env": { + "mocha": true + }, + "rules": { + // Swap the no-unused-expressions rule with a more chai-friendly one + "no-unused-expressions": 0, + "chai-friendly/no-unused-expressions": "error" + }, + "overrides": [ + { + // Test specific rules + "files": ["**/test/*/src/**/*.js"], + "globals": { + "expect": true + }, + "rules": { + // mocha-specific rules + "mocha/handle-done-callback": "error", + "mocha/no-exclusive-tests": "error", + "mocha/no-global-tests": "error", + "mocha/no-identical-title": "error", + "mocha/no-nested-tests": "error", + "mocha/no-pending-tests": "error", + "mocha/no-skipped-tests": "error", + "mocha/no-mocha-arrows": "error", + + // chai-specific rules + "chai-expect/missing-assertion": "error", + "chai-expect/terminating-properties": "error", + + // prefer-arrow-callback applies to all callbacks, not just ones in mocha tests. + // we don't enforce this at the top-level - just in tests to manage `this` scope + // based on mocha's context mechanism + "mocha/prefer-arrow-callback": "error" + } + }, + { + // Frontend test specific rules + "files": ["**/test/frontend/**/*.js"], + "globals": { + "expect": true, + "define": true, + "$": true + } + }, + { + // Backend specific rules + "files": ["**/app/src/**/*.js"], + "rules": { + // don't allow console.log in backend code + "no-console": "error" + } + } + ] +} diff --git a/services/filestore/.prettierrc b/services/filestore/.prettierrc new file mode 100644 index 0000000000..b2095be81e --- /dev/null +++ b/services/filestore/.prettierrc @@ -0,0 +1,4 @@ +{ + "semi": false, + "singleQuote": true +} diff --git a/services/filestore/npm-shrinkwrap.json b/services/filestore/npm-shrinkwrap.json index 413c41dca4..64529d81ca 100644 --- a/services/filestore/npm-shrinkwrap.json +++ b/services/filestore/npm-shrinkwrap.json @@ -583,6 +583,14 @@ "requires": { "lodash.unescape": "4.0.1", "semver": "5.5.0" + }, + "dependencies": { + "semver": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.5.0.tgz", + "integrity": "sha512-4SJ3dm0WAwWy/NVeioZh5AntkdJoWKxHxcmyP622fOkgHa4z3R0TdBJICINyaSDE6uNwVc8gZr+ZinwZAH4xIA==", + "dev": true + } } }, "abort-controller": { @@ -1184,6 +1192,14 @@ "semver": "^5.5.0", "shebang-command": "^1.2.0", "which": "^1.2.9" + }, + "dependencies": { + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true + } } }, "cryptiles": { @@ -1485,6 +1501,35 @@ "table": "^5.2.3", "text-table": "^0.2.0", "v8-compile-cache": "^2.0.3" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "lodash": { + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", + "dev": true + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + } } }, "eslint-config-prettier": { @@ -1581,6 +1626,15 @@ "esutils": "^2.0.2", "isarray": "^1.0.0" } + }, + "resolve": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.13.1.tgz", + "integrity": "sha512-CxqObCX8K8YtAhOBRg+lrcdn+LK+WYOS8tSjqSFbjtrI5PnS63QPhZl4+yKfrU9tdsbMu9Anr/amegT87M9Z6w==", + "dev": true, + "requires": { + "path-parse": "^1.0.6" + } } } }, @@ -1670,6 +1724,14 @@ "acorn": "^7.1.0", "acorn-jsx": "^5.1.0", "eslint-visitor-keys": "^1.1.0" + }, + "dependencies": { + "acorn": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.1.0.tgz", + "integrity": "sha512-kL5CuoXA/dgxlBbVrflsflzQ3PAas7RYZB52NOm/6839iVYJgKMJ3cQJD+t2i5+qFa8h3MDpEOJiS64E8JLnSQ==", + "dev": true + } } }, "esprima": { @@ -1842,6 +1904,17 @@ "chardet": "^0.7.0", "iconv-lite": "^0.4.24", "tmp": "^0.0.33" + }, + "dependencies": { + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dev": true, + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + } } }, "extsprintf": { @@ -1943,6 +2016,31 @@ "flatted": "^2.0.0", "rimraf": "2.6.3", "write": "1.0.3" + }, + "dependencies": { + "glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + } } }, "flatted": { @@ -2083,6 +2181,7 @@ "version": "6.0.4", "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", "integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=", + "optional": true, "requires": { "inflight": "^1.0.4", "inherits": "2", @@ -2372,6 +2471,14 @@ "string-width": "^4.1.0", "strip-ansi": "^5.1.0", "through": "^2.3.6" + }, + "dependencies": { + "lodash": { + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", + "dev": true + } } }, "ipaddr.js": { @@ -2628,6 +2735,14 @@ "parse-json": "^2.2.0", "pify": "^2.0.0", "strip-bom": "^3.0.0" + }, + "dependencies": { + "pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "dev": true + } } }, "locate-path": { @@ -2839,6 +2954,12 @@ "requires": { "ansi-regex": "^2.0.0" } + }, + "supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=", + "dev": true } } }, @@ -2882,6 +3003,15 @@ "dev": true, "requires": { "minimist": "^1.2.0" + }, + "dependencies": { + "minimist": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", + "dev": true, + "optional": true + } } }, "map-obj": { @@ -3264,6 +3394,14 @@ "resolve": "^1.10.0", "semver": "2 || 3 || 4 || 5", "validate-npm-package-license": "^3.0.1" + }, + "dependencies": { + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true + } } }, "oauth-sign": { @@ -3367,6 +3505,23 @@ "dev": true, "requires": { "p-limit": "^1.1.0" + }, + "dependencies": { + "p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, + "requires": { + "p-try": "^1.0.0" + } + }, + "p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", + "dev": true + } } }, "p-try": { @@ -3447,6 +3602,14 @@ "dev": true, "requires": { "pify": "^2.0.0" + }, + "dependencies": { + "pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "dev": true + } } }, "pathval": { @@ -3536,6 +3699,15 @@ "restore-cursor": "^2.0.0" } }, + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, "eslint": { "version": "5.16.0", "resolved": "https://registry.npmjs.org/eslint/-/eslint-5.16.0.tgz", @@ -3610,6 +3782,20 @@ "escape-string-regexp": "^1.0.5" } }, + "glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, "globals": { "version": "11.12.0", "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", @@ -3643,6 +3829,12 @@ "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", "dev": true }, + "lodash": { + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", + "dev": true + }, "strip-ansi": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", @@ -3666,6 +3858,12 @@ "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", "dev": true }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, "mute-stream": { "version": "0.0.7", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", @@ -3691,6 +3889,12 @@ "signal-exit": "^3.0.2" } }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true + }, "string-width": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", @@ -3764,6 +3968,15 @@ "restore-cursor": "^2.0.0" } }, + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, "eslint": { "version": "5.16.0", "resolved": "https://registry.npmjs.org/eslint/-/eslint-5.16.0.tgz", @@ -3862,6 +4075,20 @@ "integrity": "sha512-zRKcywvrXlXsA0v0i9Io4KDRaAw7+a1ZpjRwl9Wox8PFlVCCHra7E9c4kqXCoCM9nR5tBkaTTZRBoCm60bFqTQ==", "dev": true }, + "glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, "globals": { "version": "11.12.0", "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", @@ -3901,6 +4128,12 @@ "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", "dev": true }, + "lodash": { + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", + "dev": true + }, "strip-ansi": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", @@ -3933,6 +4166,12 @@ "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", "dev": true }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, "mute-stream": { "version": "0.0.7", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", @@ -3973,6 +4212,12 @@ "signal-exit": "^3.0.2" } }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true + }, "string-width": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", @@ -4779,6 +5024,18 @@ "string-width": "^3.0.0" }, "dependencies": { + "ajv": { + "version": "6.10.2", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.2.tgz", + "integrity": "sha512-TXtUUEYHuaTEbLZWIKUr5pmBuhDLy+8KYtPYdcV8qC+pOZL+NKqYwvWSRrVXHn+ZmRRAu8vJTAznH7Oag6RVRw==", + "dev": true, + "requires": { + "fast-deep-equal": "^2.0.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, "emoji-regex": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", @@ -4791,6 +5048,12 @@ "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", "dev": true }, + "lodash": { + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", + "dev": true + }, "string-width": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", @@ -5045,6 +5308,12 @@ "lodash": "^4.17.4" }, "dependencies": { + "acorn": { + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.3.tgz", + "integrity": "sha512-T/zvzYRfbVojPWahDsE5evJdHb3oJoQfFbsrKM7w5Zcs++Tr257tia3BmMP8XYVjp1S9RZXQMh7gao96BlqZOw==", + "dev": true + }, "acorn-jsx": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-3.0.1.tgz", @@ -5052,6 +5321,23 @@ "dev": true, "requires": { "acorn": "^3.0.4" + }, + "dependencies": { + "acorn": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-3.3.0.tgz", + "integrity": "sha1-ReN/s56No/JbruP/U2niu18iAXo=", + "dev": true + } + } + }, + "debug": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "dev": true, + "requires": { + "ms": "^2.1.1" } }, "eslint-scope": { @@ -5073,6 +5359,12 @@ "acorn": "^5.5.0", "acorn-jsx": "^3.0.0" } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true } } }, From 8d2c87420e0c48e477f6b89f397f912bb1d38d00 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Mon, 16 Dec 2019 10:24:32 +0000 Subject: [PATCH 06/73] decaffeinate: Rename AWSSDKPersistorManager.coffee and 13 other files from .coffee to .js --- .../{AWSSDKPersistorManager.coffee => AWSSDKPersistorManager.js} | 0 .../app/coffee/{BucketController.coffee => BucketController.js} | 0 services/filestore/app/coffee/{Errors.coffee => Errors.js} | 0 .../coffee/{FSPersistorManager.coffee => FSPersistorManager.js} | 0 .../app/coffee/{FileController.coffee => FileController.js} | 0 .../app/coffee/{FileConverter.coffee => FileConverter.js} | 0 .../filestore/app/coffee/{FileHandler.coffee => FileHandler.js} | 0 .../{HealthCheckController.coffee => HealthCheckController.js} | 0 .../app/coffee/{ImageOptimiser.coffee => ImageOptimiser.js} | 0 .../filestore/app/coffee/{KeyBuilder.coffee => KeyBuilder.js} | 0 .../app/coffee/{LocalFileWriter.coffee => LocalFileWriter.js} | 0 .../app/coffee/{PersistorManager.coffee => PersistorManager.js} | 0 .../coffee/{S3PersistorManager.coffee => S3PersistorManager.js} | 0 services/filestore/app/coffee/{SafeExec.coffee => SafeExec.js} | 0 14 files changed, 0 insertions(+), 0 deletions(-) rename services/filestore/app/coffee/{AWSSDKPersistorManager.coffee => AWSSDKPersistorManager.js} (100%) rename services/filestore/app/coffee/{BucketController.coffee => BucketController.js} (100%) rename services/filestore/app/coffee/{Errors.coffee => Errors.js} (100%) rename services/filestore/app/coffee/{FSPersistorManager.coffee => FSPersistorManager.js} (100%) rename services/filestore/app/coffee/{FileController.coffee => FileController.js} (100%) rename services/filestore/app/coffee/{FileConverter.coffee => FileConverter.js} (100%) rename services/filestore/app/coffee/{FileHandler.coffee => FileHandler.js} (100%) rename services/filestore/app/coffee/{HealthCheckController.coffee => HealthCheckController.js} (100%) rename services/filestore/app/coffee/{ImageOptimiser.coffee => ImageOptimiser.js} (100%) rename services/filestore/app/coffee/{KeyBuilder.coffee => KeyBuilder.js} (100%) rename services/filestore/app/coffee/{LocalFileWriter.coffee => LocalFileWriter.js} (100%) rename services/filestore/app/coffee/{PersistorManager.coffee => PersistorManager.js} (100%) rename services/filestore/app/coffee/{S3PersistorManager.coffee => S3PersistorManager.js} (100%) rename services/filestore/app/coffee/{SafeExec.coffee => SafeExec.js} (100%) diff --git a/services/filestore/app/coffee/AWSSDKPersistorManager.coffee b/services/filestore/app/coffee/AWSSDKPersistorManager.js similarity index 100% rename from services/filestore/app/coffee/AWSSDKPersistorManager.coffee rename to services/filestore/app/coffee/AWSSDKPersistorManager.js diff --git a/services/filestore/app/coffee/BucketController.coffee b/services/filestore/app/coffee/BucketController.js similarity index 100% rename from services/filestore/app/coffee/BucketController.coffee rename to services/filestore/app/coffee/BucketController.js diff --git a/services/filestore/app/coffee/Errors.coffee b/services/filestore/app/coffee/Errors.js similarity index 100% rename from services/filestore/app/coffee/Errors.coffee rename to services/filestore/app/coffee/Errors.js diff --git a/services/filestore/app/coffee/FSPersistorManager.coffee b/services/filestore/app/coffee/FSPersistorManager.js similarity index 100% rename from services/filestore/app/coffee/FSPersistorManager.coffee rename to services/filestore/app/coffee/FSPersistorManager.js diff --git a/services/filestore/app/coffee/FileController.coffee b/services/filestore/app/coffee/FileController.js similarity index 100% rename from services/filestore/app/coffee/FileController.coffee rename to services/filestore/app/coffee/FileController.js diff --git a/services/filestore/app/coffee/FileConverter.coffee b/services/filestore/app/coffee/FileConverter.js similarity index 100% rename from services/filestore/app/coffee/FileConverter.coffee rename to services/filestore/app/coffee/FileConverter.js diff --git a/services/filestore/app/coffee/FileHandler.coffee b/services/filestore/app/coffee/FileHandler.js similarity index 100% rename from services/filestore/app/coffee/FileHandler.coffee rename to services/filestore/app/coffee/FileHandler.js diff --git a/services/filestore/app/coffee/HealthCheckController.coffee b/services/filestore/app/coffee/HealthCheckController.js similarity index 100% rename from services/filestore/app/coffee/HealthCheckController.coffee rename to services/filestore/app/coffee/HealthCheckController.js diff --git a/services/filestore/app/coffee/ImageOptimiser.coffee b/services/filestore/app/coffee/ImageOptimiser.js similarity index 100% rename from services/filestore/app/coffee/ImageOptimiser.coffee rename to services/filestore/app/coffee/ImageOptimiser.js diff --git a/services/filestore/app/coffee/KeyBuilder.coffee b/services/filestore/app/coffee/KeyBuilder.js similarity index 100% rename from services/filestore/app/coffee/KeyBuilder.coffee rename to services/filestore/app/coffee/KeyBuilder.js diff --git a/services/filestore/app/coffee/LocalFileWriter.coffee b/services/filestore/app/coffee/LocalFileWriter.js similarity index 100% rename from services/filestore/app/coffee/LocalFileWriter.coffee rename to services/filestore/app/coffee/LocalFileWriter.js diff --git a/services/filestore/app/coffee/PersistorManager.coffee b/services/filestore/app/coffee/PersistorManager.js similarity index 100% rename from services/filestore/app/coffee/PersistorManager.coffee rename to services/filestore/app/coffee/PersistorManager.js diff --git a/services/filestore/app/coffee/S3PersistorManager.coffee b/services/filestore/app/coffee/S3PersistorManager.js similarity index 100% rename from services/filestore/app/coffee/S3PersistorManager.coffee rename to services/filestore/app/coffee/S3PersistorManager.js diff --git a/services/filestore/app/coffee/SafeExec.coffee b/services/filestore/app/coffee/SafeExec.js similarity index 100% rename from services/filestore/app/coffee/SafeExec.coffee rename to services/filestore/app/coffee/SafeExec.js From 6bd8452f19b3a8c82e7330e804cac94c25fd9ae1 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Mon, 16 Dec 2019 10:24:35 +0000 Subject: [PATCH 07/73] decaffeinate: Convert AWSSDKPersistorManager.coffee and 13 other files to JS --- .../app/coffee/AWSSDKPersistorManager.js | 225 +++++---- .../filestore/app/coffee/BucketController.js | 66 ++- services/filestore/app/coffee/Errors.js | 18 +- .../app/coffee/FSPersistorManager.js | 270 +++++----- .../filestore/app/coffee/FileController.js | 238 +++++---- .../filestore/app/coffee/FileConverter.js | 126 +++-- services/filestore/app/coffee/FileHandler.js | 268 +++++----- .../app/coffee/HealthCheckController.js | 119 +++-- .../filestore/app/coffee/ImageOptimiser.js | 58 ++- services/filestore/app/coffee/KeyBuilder.js | 98 ++-- .../filestore/app/coffee/LocalFileWriter.js | 122 +++-- .../filestore/app/coffee/PersistorManager.js | 40 +- .../app/coffee/S3PersistorManager.js | 460 ++++++++++-------- services/filestore/app/coffee/SafeExec.js | 88 ++-- 14 files changed, 1260 insertions(+), 936 deletions(-) diff --git a/services/filestore/app/coffee/AWSSDKPersistorManager.js b/services/filestore/app/coffee/AWSSDKPersistorManager.js index 168fc68d54..c263936073 100644 --- a/services/filestore/app/coffee/AWSSDKPersistorManager.js +++ b/services/filestore/app/coffee/AWSSDKPersistorManager.js @@ -1,106 +1,145 @@ -# This module is not used in production, which currently uses -# S3PersistorManager. The intention is to migrate S3PersistorManager to use the -# latest aws-sdk and delete this module so that PersistorManager would load the -# same backend for both the 's3' and 'aws-sdk' options. +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +// This module is not used in production, which currently uses +// S3PersistorManager. The intention is to migrate S3PersistorManager to use the +// latest aws-sdk and delete this module so that PersistorManager would load the +// same backend for both the 's3' and 'aws-sdk' options. -logger = require "logger-sharelatex" -aws = require "aws-sdk" -_ = require "underscore" -fs = require "fs" -Errors = require "./Errors" +const logger = require("logger-sharelatex"); +const aws = require("aws-sdk"); +const _ = require("underscore"); +const fs = require("fs"); +const Errors = require("./Errors"); -s3 = new aws.S3() +const s3 = new aws.S3(); -module.exports = - sendFile: (bucketName, key, fsPath, callback)-> - logger.log bucketName:bucketName, key:key, "send file data to s3" - stream = fs.createReadStream fsPath - s3.upload Bucket: bucketName, Key: key, Body: stream, (err, data) -> - if err? - logger.err err: err, Bucket: bucketName, Key: key, "error sending file data to s3" - callback err +module.exports = { + sendFile(bucketName, key, fsPath, callback){ + logger.log({bucketName, key}, "send file data to s3"); + const stream = fs.createReadStream(fsPath); + return s3.upload({Bucket: bucketName, Key: key, Body: stream}, function(err, data) { + if (err != null) { + logger.err({err, Bucket: bucketName, Key: key}, "error sending file data to s3"); + } + return callback(err); + }); + }, - sendStream: (bucketName, key, stream, callback)-> - logger.log bucketName:bucketName, key:key, "send file stream to s3" - s3.upload Bucket: bucketName, Key: key, Body: stream, (err, data) -> - if err? - logger.err err: err, Bucket: bucketName, Key: key, "error sending file stream to s3" - callback err + sendStream(bucketName, key, stream, callback){ + logger.log({bucketName, key}, "send file stream to s3"); + return s3.upload({Bucket: bucketName, Key: key, Body: stream}, function(err, data) { + if (err != null) { + logger.err({err, Bucket: bucketName, Key: key}, "error sending file stream to s3"); + } + return callback(err); + }); + }, - getFileStream: (bucketName, key, opts, callback = (err, res)->)-> - logger.log bucketName:bucketName, key:key, "get file stream from s3" - callback = _.once callback - params = - Bucket:bucketName + getFileStream(bucketName, key, opts, callback){ + if (callback == null) { callback = function(err, res){}; } + logger.log({bucketName, key}, "get file stream from s3"); + callback = _.once(callback); + const params = { + Bucket:bucketName, Key: key - if opts.start? and opts.end? - params['Range'] = "bytes=#{opts.start}-#{opts.end}" - request = s3.getObject params - stream = request.createReadStream() - stream.on 'readable', () -> - callback null, stream - stream.on 'error', (err) -> - logger.err err:err, bucketName:bucketName, key:key, "error getting file stream from s3" - if err.code == 'NoSuchKey' - return callback new Errors.NotFoundError "File not found in S3: #{bucketName}:#{key}" - callback err + }; + if ((opts.start != null) && (opts.end != null)) { + params['Range'] = `bytes=${opts.start}-${opts.end}`; + } + const request = s3.getObject(params); + const stream = request.createReadStream(); + stream.on('readable', () => callback(null, stream)); + return stream.on('error', function(err) { + logger.err({err, bucketName, key}, "error getting file stream from s3"); + if (err.code === 'NoSuchKey') { + return callback(new Errors.NotFoundError(`File not found in S3: ${bucketName}:${key}`)); + } + return callback(err); + }); + }, - copyFile: (bucketName, sourceKey, destKey, callback)-> - logger.log bucketName:bucketName, sourceKey:sourceKey, destKey: destKey, "copying file in s3" - source = bucketName + '/' + sourceKey - s3.copyObject {Bucket: bucketName, Key: destKey, CopySource: source}, (err) -> - if err? - logger.err err:err, bucketName:bucketName, sourceKey:sourceKey, destKey:destKey, "something went wrong copying file in s3" - callback err + copyFile(bucketName, sourceKey, destKey, callback){ + logger.log({bucketName, sourceKey, destKey}, "copying file in s3"); + const source = bucketName + '/' + sourceKey; + return s3.copyObject({Bucket: bucketName, Key: destKey, CopySource: source}, function(err) { + if (err != null) { + logger.err({err, bucketName, sourceKey, destKey}, "something went wrong copying file in s3"); + } + return callback(err); + }); + }, - deleteFile: (bucketName, key, callback)-> - logger.log bucketName:bucketName, key:key, "delete file in s3" - s3.deleteObject {Bucket: bucketName, Key: key}, (err) -> - if err? - logger.err err:err, bucketName:bucketName, key:key, "something went wrong deleting file in s3" - callback err + deleteFile(bucketName, key, callback){ + logger.log({bucketName, key}, "delete file in s3"); + return s3.deleteObject({Bucket: bucketName, Key: key}, function(err) { + if (err != null) { + logger.err({err, bucketName, key}, "something went wrong deleting file in s3"); + } + return callback(err); + }); + }, - deleteDirectory: (bucketName, key, callback)-> - logger.log bucketName:bucketName, key:key, "delete directory in s3" - s3.listObjects {Bucket: bucketName, Prefix: key}, (err, data) -> - if err? - logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in s3" - return callback err - if data.Contents.length == 0 - logger.log bucketName:bucketName, key:key, "the directory is empty" - return callback() - keys = _.map data.Contents, (entry)-> - Key: entry.Key - s3.deleteObjects - Bucket: bucketName - Delete: - Objects: keys + deleteDirectory(bucketName, key, callback){ + logger.log({bucketName, key}, "delete directory in s3"); + return s3.listObjects({Bucket: bucketName, Prefix: key}, function(err, data) { + if (err != null) { + logger.err({err, bucketName, key}, "something went wrong listing prefix in s3"); + return callback(err); + } + if (data.Contents.length === 0) { + logger.log({bucketName, key}, "the directory is empty"); + return callback(); + } + const keys = _.map(data.Contents, entry => ({ + Key: entry.Key + })); + return s3.deleteObjects({ + Bucket: bucketName, + Delete: { + Objects: keys, Quiet: true - , (err) -> - if err? - logger.err err:err, bucketName:bucketName, key:keys, "something went wrong deleting directory in s3" - callback err + } + } + , function(err) { + if (err != null) { + logger.err({err, bucketName, key:keys}, "something went wrong deleting directory in s3"); + } + return callback(err); + }); + }); + }, - checkIfFileExists:(bucketName, key, callback)-> - logger.log bucketName:bucketName, key:key, "check file existence in s3" - s3.headObject {Bucket: bucketName, Key: key}, (err, data) -> - if err? - return (callback null, false) if err.code == 'NotFound' - logger.err err:err, bucketName:bucketName, key:key, "something went wrong checking head in s3" - return callback err - callback null, data.ETag? + checkIfFileExists(bucketName, key, callback){ + logger.log({bucketName, key}, "check file existence in s3"); + return s3.headObject({Bucket: bucketName, Key: key}, function(err, data) { + if (err != null) { + if (err.code === 'NotFound') { return (callback(null, false)); } + logger.err({err, bucketName, key}, "something went wrong checking head in s3"); + return callback(err); + } + return callback(null, (data.ETag != null)); + }); + }, - directorySize:(bucketName, key, callback)-> - logger.log bucketName:bucketName, key:key, "get project size in s3" - s3.listObjects {Bucket: bucketName, Prefix: key}, (err, data) -> - if err? - logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in s3" - return callback err - if data.Contents.length == 0 - logger.log bucketName:bucketName, key:key, "the directory is empty" - return callback() - totalSize = 0 - _.each data.Contents, (entry)-> - totalSize += entry.Size - callback null, totalSize + directorySize(bucketName, key, callback){ + logger.log({bucketName, key}, "get project size in s3"); + return s3.listObjects({Bucket: bucketName, Prefix: key}, function(err, data) { + if (err != null) { + logger.err({err, bucketName, key}, "something went wrong listing prefix in s3"); + return callback(err); + } + if (data.Contents.length === 0) { + logger.log({bucketName, key}, "the directory is empty"); + return callback(); + } + let totalSize = 0; + _.each(data.Contents, entry => totalSize += entry.Size); + return callback(null, totalSize); + }); + } +}; diff --git a/services/filestore/app/coffee/BucketController.js b/services/filestore/app/coffee/BucketController.js index bb7bd4544b..ecd4f9a54f 100644 --- a/services/filestore/app/coffee/BucketController.js +++ b/services/filestore/app/coffee/BucketController.js @@ -1,29 +1,41 @@ -settings = require("settings-sharelatex") -logger = require("logger-sharelatex") -FileHandler = require("./FileHandler") -metrics = require("metrics-sharelatex") -Errors = require('./Errors') +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let BucketController; +const settings = require("settings-sharelatex"); +const logger = require("logger-sharelatex"); +const FileHandler = require("./FileHandler"); +const metrics = require("metrics-sharelatex"); +const Errors = require('./Errors'); -module.exports = BucketController = +module.exports = (BucketController = { - getFile: (req, res)-> - {bucket} = req.params - key = req.params[0] - credentials = settings.filestore.s3BucketCreds?[bucket] - options = { - key: key, - bucket: bucket, - credentials: credentials - } - metrics.inc "#{bucket}.getFile" - logger.log key:key, bucket:bucket, "receiving request to get file from bucket" - FileHandler.getFile bucket, key, options, (err, fileStream)-> - if err? - logger.err err:err, key:key, bucket:bucket, "problem getting file from bucket" - if err instanceof Errors.NotFoundError - return res.send 404 - else - return res.send 500 - else - logger.log key:key, bucket:bucket, "sending bucket file to response" - fileStream.pipe res + getFile(req, res){ + const {bucket} = req.params; + const key = req.params[0]; + const credentials = settings.filestore.s3BucketCreds != null ? settings.filestore.s3BucketCreds[bucket] : undefined; + const options = { + key, + bucket, + credentials + }; + metrics.inc(`${bucket}.getFile`); + logger.log({key, bucket}, "receiving request to get file from bucket"); + return FileHandler.getFile(bucket, key, options, function(err, fileStream){ + if (err != null) { + logger.err({err, key, bucket}, "problem getting file from bucket"); + if (err instanceof Errors.NotFoundError) { + return res.send(404); + } else { + return res.send(500); + } + } else { + logger.log({key, bucket}, "sending bucket file to response"); + return fileStream.pipe(res); + } + }); + } +}); diff --git a/services/filestore/app/coffee/Errors.js b/services/filestore/app/coffee/Errors.js index 3bd9479abe..c4f8f7004c 100644 --- a/services/filestore/app/coffee/Errors.js +++ b/services/filestore/app/coffee/Errors.js @@ -1,9 +1,11 @@ -NotFoundError = (message) -> - error = new Error(message) - error.name = "NotFoundError" - error.__proto__ = NotFoundError.prototype - return error -NotFoundError.prototype.__proto__ = Error.prototype +let Errors; +var NotFoundError = function(message) { + const error = new Error(message); + error.name = "NotFoundError"; + error.__proto__ = NotFoundError.prototype; + return error; +}; +NotFoundError.prototype.__proto__ = Error.prototype; -module.exports = Errors = - NotFoundError: NotFoundError +module.exports = (Errors = + {NotFoundError}); diff --git a/services/filestore/app/coffee/FSPersistorManager.js b/services/filestore/app/coffee/FSPersistorManager.js index 38e30f284f..c4792e6215 100644 --- a/services/filestore/app/coffee/FSPersistorManager.js +++ b/services/filestore/app/coffee/FSPersistorManager.js @@ -1,128 +1,164 @@ -logger = require("logger-sharelatex") -fs = require("fs") -path = require("path") -LocalFileWriter = require("./LocalFileWriter") -Errors = require('./Errors') -rimraf = require("rimraf") -_ = require "underscore" +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const logger = require("logger-sharelatex"); +const fs = require("fs"); +const path = require("path"); +const LocalFileWriter = require("./LocalFileWriter"); +const Errors = require('./Errors'); +const rimraf = require("rimraf"); +const _ = require("underscore"); -filterName = (key) -> - return key.replace /\//g, "_" +const filterName = key => key.replace(/\//g, "_"); -module.exports = - sendFile: ( location, target, source, callback = (err)->) -> - filteredTarget = filterName target - logger.log location:location, target:filteredTarget, source:source, "sending file" - done = _.once (err) -> - if err? - logger.err err:err, location:location, target:filteredTarget, source:source, "Error on put of file" - callback(err) - # actually copy the file (instead of moving it) to maintain consistent behaviour - # between the different implementations - sourceStream = fs.createReadStream source - sourceStream.on 'error', done - targetStream = fs.createWriteStream "#{location}/#{filteredTarget}" - targetStream.on 'error', done - targetStream.on 'finish', () -> - done() - sourceStream.pipe targetStream +module.exports = { + sendFile( location, target, source, callback) { + if (callback == null) { callback = function(err){}; } + const filteredTarget = filterName(target); + logger.log({location, target:filteredTarget, source}, "sending file"); + const done = _.once(function(err) { + if (err != null) { + logger.err({err, location, target:filteredTarget, source}, "Error on put of file"); + } + return callback(err); + }); + // actually copy the file (instead of moving it) to maintain consistent behaviour + // between the different implementations + const sourceStream = fs.createReadStream(source); + sourceStream.on('error', done); + const targetStream = fs.createWriteStream(`${location}/${filteredTarget}`); + targetStream.on('error', done); + targetStream.on('finish', () => done()); + return sourceStream.pipe(targetStream); + }, - sendStream: ( location, target, sourceStream, callback = (err)->) -> - logger.log location:location, target:target, "sending file stream" - sourceStream.on "error", (err)-> - logger.err location:location, target:target, err:err "error on stream to send" - LocalFileWriter.writeStream sourceStream, null, (err, fsPath)=> - if err? - logger.err location:location, target:target, fsPath:fsPath, err:err, "something went wrong writing stream to disk" - return callback err - @sendFile location, target, fsPath, (err) -> - # delete the temporary file created above and return the original error - LocalFileWriter.deleteFile fsPath, () -> - callback(err) + sendStream( location, target, sourceStream, callback) { + if (callback == null) { callback = function(err){}; } + logger.log({location, target}, "sending file stream"); + sourceStream.on("error", err => logger.err({location, target, err:err("error on stream to send")})); + return LocalFileWriter.writeStream(sourceStream, null, (err, fsPath)=> { + if (err != null) { + logger.err({location, target, fsPath, err}, "something went wrong writing stream to disk"); + return callback(err); + } + return this.sendFile(location, target, fsPath, err => // delete the temporary file created above and return the original error + LocalFileWriter.deleteFile(fsPath, () => callback(err))); + }); + }, - # opts may be {start: Number, end: Number} - getFileStream: (location, name, opts, callback = (err, res)->) -> - filteredName = filterName name - logger.log location:location, filteredName:filteredName, "getting file" - fs.open "#{location}/#{filteredName}", 'r', (err, fd) -> - if err? - logger.err err:err, location:location, filteredName:name, "Error reading from file" - if err.code == 'ENOENT' - return callback new Errors.NotFoundError(err.message), null - else - return callback err, null - opts.fd = fd - sourceStream = fs.createReadStream null, opts - return callback null, sourceStream + // opts may be {start: Number, end: Number} + getFileStream(location, name, opts, callback) { + if (callback == null) { callback = function(err, res){}; } + const filteredName = filterName(name); + logger.log({location, filteredName}, "getting file"); + return fs.open(`${location}/${filteredName}`, 'r', function(err, fd) { + if (err != null) { + logger.err({err, location, filteredName:name}, "Error reading from file"); + } + if (err.code === 'ENOENT') { + return callback(new Errors.NotFoundError(err.message), null); + } else { + return callback(err, null); + } + opts.fd = fd; + const sourceStream = fs.createReadStream(null, opts); + return callback(null, sourceStream); + }); + }, - getFileSize: (location, filename, callback) -> - fullPath = path.join(location, filterName(filename)) - fs.stat fullPath, (err, stats) -> - if err? - if err.code == 'ENOENT' - logger.log({location:location, filename:filename}, "file not found") - callback(new Errors.NotFoundError(err.message)) - else - logger.err({err:err, location:location, filename:filename}, "failed to stat file") - callback(err) - return - callback(null, stats.size) + getFileSize(location, filename, callback) { + const fullPath = path.join(location, filterName(filename)); + return fs.stat(fullPath, function(err, stats) { + if (err != null) { + if (err.code === 'ENOENT') { + logger.log({location, filename}, "file not found"); + callback(new Errors.NotFoundError(err.message)); + } else { + logger.err({err, location, filename}, "failed to stat file"); + callback(err); + } + return; + } + return callback(null, stats.size); + }); + }, - copyFile: (location, fromName, toName, callback = (err)->)-> - filteredFromName=filterName fromName - filteredToName=filterName toName - logger.log location:location, fromName:filteredFromName, toName:filteredToName, "copying file" - sourceStream = fs.createReadStream "#{location}/#{filteredFromName}" - sourceStream.on 'error', (err) -> - logger.err err:err, location:location, key:filteredFromName, "Error reading from file" - callback err - targetStream = fs.createWriteStream "#{location}/#{filteredToName}" - targetStream.on 'error', (err) -> - logger.err err:err, location:location, key:filteredToName, "Error writing to file" - callback err - targetStream.on 'finish', () -> - callback null - sourceStream.pipe targetStream + copyFile(location, fromName, toName, callback){ + if (callback == null) { callback = function(err){}; } + const filteredFromName=filterName(fromName); + const filteredToName=filterName(toName); + logger.log({location, fromName:filteredFromName, toName:filteredToName}, "copying file"); + const sourceStream = fs.createReadStream(`${location}/${filteredFromName}`); + sourceStream.on('error', function(err) { + logger.err({err, location, key:filteredFromName}, "Error reading from file"); + return callback(err); + }); + const targetStream = fs.createWriteStream(`${location}/${filteredToName}`); + targetStream.on('error', function(err) { + logger.err({err, location, key:filteredToName}, "Error writing to file"); + return callback(err); + }); + targetStream.on('finish', () => callback(null)); + return sourceStream.pipe(targetStream); + }, - deleteFile: (location, name, callback)-> - filteredName = filterName name - logger.log location:location, filteredName:filteredName, "delete file" - fs.unlink "#{location}/#{filteredName}", (err) -> - if err? - logger.err err:err, location:location, filteredName:filteredName, "Error on delete." - callback err - else - callback() + deleteFile(location, name, callback){ + const filteredName = filterName(name); + logger.log({location, filteredName}, "delete file"); + return fs.unlink(`${location}/${filteredName}`, function(err) { + if (err != null) { + logger.err({err, location, filteredName}, "Error on delete."); + return callback(err); + } else { + return callback(); + } + }); + }, - deleteDirectory: (location, name, callback = (err)->)-> - filteredName = filterName name.replace(/\/$/,'') - rimraf "#{location}/#{filteredName}", (err) -> - if err? - logger.err err:err, location:location, filteredName:filteredName, "Error on rimraf rmdir." - callback err - else - callback() + deleteDirectory(location, name, callback){ + if (callback == null) { callback = function(err){}; } + const filteredName = filterName(name.replace(/\/$/,'')); + return rimraf(`${location}/${filteredName}`, function(err) { + if (err != null) { + logger.err({err, location, filteredName}, "Error on rimraf rmdir."); + return callback(err); + } else { + return callback(); + } + }); + }, - checkIfFileExists:(location, name, callback = (err,exists)->)-> - filteredName = filterName name - logger.log location:location, filteredName:filteredName, "checking if file exists" - fs.exists "#{location}/#{filteredName}", (exists) -> - logger.log location:location, filteredName:filteredName, exists:exists, "checked if file exists" - callback null, exists + checkIfFileExists(location, name, callback){ + if (callback == null) { callback = function(err,exists){}; } + const filteredName = filterName(name); + logger.log({location, filteredName}, "checking if file exists"); + return fs.exists(`${location}/${filteredName}`, function(exists) { + logger.log({location, filteredName, exists}, "checked if file exists"); + return callback(null, exists); + }); + }, - directorySize:(location, name, callback)-> - filteredName = filterName name.replace(/\/$/,'') - logger.log location:location, filteredName:filteredName, "get project size in file system" - fs.readdir "#{location}/#{filteredName}", (err, files) -> - if err? - logger.err err:err, location:location, filteredName:filteredName, "something went wrong listing prefix in aws" - return callback(err) - totalSize = 0 - _.each files, (entry)-> - fd = fs.openSync "#{location}/#{filteredName}/#{entry}", 'r' - fileStats = fs.fstatSync(fd) - totalSize += fileStats.size - fs.closeSync fd - logger.log totalSize:totalSize, "total size", files:files - callback null, totalSize + directorySize(location, name, callback){ + const filteredName = filterName(name.replace(/\/$/,'')); + logger.log({location, filteredName}, "get project size in file system"); + return fs.readdir(`${location}/${filteredName}`, function(err, files) { + if (err != null) { + logger.err({err, location, filteredName}, "something went wrong listing prefix in aws"); + return callback(err); + } + let totalSize = 0; + _.each(files, function(entry){ + const fd = fs.openSync(`${location}/${filteredName}/${entry}`, 'r'); + const fileStats = fs.fstatSync(fd); + totalSize += fileStats.size; + return fs.closeSync(fd); + }); + logger.log({totalSize}, "total size", {files}); + return callback(null, totalSize); + }); + } +}; diff --git a/services/filestore/app/coffee/FileController.js b/services/filestore/app/coffee/FileController.js index f98dbd1e49..f40d4a0317 100644 --- a/services/filestore/app/coffee/FileController.js +++ b/services/filestore/app/coffee/FileController.js @@ -1,113 +1,145 @@ -PersistorManager = require("./PersistorManager") -settings = require("settings-sharelatex") -logger = require("logger-sharelatex") -FileHandler = require("./FileHandler") -metrics = require("metrics-sharelatex") -parseRange = require('range-parser') -Errors = require('./Errors') +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let FileController; +const PersistorManager = require("./PersistorManager"); +const settings = require("settings-sharelatex"); +const logger = require("logger-sharelatex"); +const FileHandler = require("./FileHandler"); +const metrics = require("metrics-sharelatex"); +const parseRange = require('range-parser'); +const Errors = require('./Errors'); -oneDayInSeconds = 60 * 60 * 24 -maxSizeInBytes = 1024 * 1024 * 1024 # 1GB +const oneDayInSeconds = 60 * 60 * 24; +const maxSizeInBytes = 1024 * 1024 * 1024; // 1GB -module.exports = FileController = +module.exports = (FileController = { - getFile: (req, res)-> - {key, bucket} = req - {format, style} = req.query - options = { - key: key, - bucket: bucket, - format: format, - style: style, + getFile(req, res){ + const {key, bucket} = req; + const {format, style} = req.query; + const options = { + key, + bucket, + format, + style, + }; + metrics.inc("getFile"); + logger.log({key, bucket, format, style}, "receiving request to get file"); + if (req.headers.range != null) { + const range = FileController._get_range(req.headers.range); + options.start = range.start; + options.end = range.end; + logger.log({start: range.start, end: range.end}, "getting range of bytes from file"); } - metrics.inc "getFile" - logger.log key:key, bucket:bucket, format:format, style: style, "receiving request to get file" - if req.headers.range? - range = FileController._get_range(req.headers.range) - options.start = range.start - options.end = range.end - logger.log start: range.start, end: range.end, "getting range of bytes from file" - FileHandler.getFile bucket, key, options, (err, fileStream)-> - if err? - if err instanceof Errors.NotFoundError - return res.send 404 - else - logger.err err:err, key:key, bucket:bucket, format:format, style:style, "problem getting file" - return res.send 500 - else if req.query.cacheWarm - logger.log key:key, bucket:bucket, format:format, style:style, "request is only for cache warm so not sending stream" - res.send 200 - else - logger.log key:key, bucket:bucket, format:format, style:style, "sending file to response" - fileStream.pipe res + return FileHandler.getFile(bucket, key, options, function(err, fileStream){ + if (err != null) { + if (err instanceof Errors.NotFoundError) { + return res.send(404); + } else { + logger.err({err, key, bucket, format, style}, "problem getting file"); + return res.send(500); + } + } else if (req.query.cacheWarm) { + logger.log({key, bucket, format, style}, "request is only for cache warm so not sending stream"); + return res.send(200); + } else { + logger.log({key, bucket, format, style}, "sending file to response"); + return fileStream.pipe(res); + } + }); + }, - getFileHead: (req, res) -> - {key, bucket} = req - metrics.inc("getFileSize") - logger.log({ key: key, bucket: bucket }, "receiving request to get file metadata") - FileHandler.getFileSize bucket, key, (err, fileSize) -> - if err? - if err instanceof Errors.NotFoundError - res.status(404).end() - else - res.status(500).end() - return - res.set("Content-Length", fileSize) - res.status(200).end() + getFileHead(req, res) { + const {key, bucket} = req; + metrics.inc("getFileSize"); + logger.log({ key, bucket }, "receiving request to get file metadata"); + return FileHandler.getFileSize(bucket, key, function(err, fileSize) { + if (err != null) { + if (err instanceof Errors.NotFoundError) { + res.status(404).end(); + } else { + res.status(500).end(); + } + return; + } + res.set("Content-Length", fileSize); + return res.status(200).end(); + }); + }, - insertFile: (req, res)-> - metrics.inc "insertFile" - {key, bucket} = req - logger.log key:key, bucket:bucket, "receiving request to insert file" - FileHandler.insertFile bucket, key, req, (err)-> - if err? - logger.log err: err, key: key, bucket: bucket, "error inserting file" - res.send 500 - else - res.send 200 + insertFile(req, res){ + metrics.inc("insertFile"); + const {key, bucket} = req; + logger.log({key, bucket}, "receiving request to insert file"); + return FileHandler.insertFile(bucket, key, req, function(err){ + if (err != null) { + logger.log({err, key, bucket}, "error inserting file"); + return res.send(500); + } else { + return res.send(200); + } + }); + }, - copyFile: (req, res)-> - metrics.inc "copyFile" - {key, bucket} = req - oldProject_id = req.body.source.project_id - oldFile_id = req.body.source.file_id - logger.log key:key, bucket:bucket, oldProject_id:oldProject_id, oldFile_id:oldFile_id, "receiving request to copy file" - PersistorManager.copyFile bucket, "#{oldProject_id}/#{oldFile_id}", key, (err)-> - if err? - if err instanceof Errors.NotFoundError - res.send 404 - else - logger.log err:err, oldProject_id:oldProject_id, oldFile_id:oldFile_id, "something went wrong copying file" - res.send 500 - else - res.send 200 + copyFile(req, res){ + metrics.inc("copyFile"); + const {key, bucket} = req; + const oldProject_id = req.body.source.project_id; + const oldFile_id = req.body.source.file_id; + logger.log({key, bucket, oldProject_id, oldFile_id}, "receiving request to copy file"); + return PersistorManager.copyFile(bucket, `${oldProject_id}/${oldFile_id}`, key, function(err){ + if (err != null) { + if (err instanceof Errors.NotFoundError) { + return res.send(404); + } else { + logger.log({err, oldProject_id, oldFile_id}, "something went wrong copying file"); + return res.send(500); + } + } else { + return res.send(200); + } + }); + }, - deleteFile: (req, res)-> - metrics.inc "deleteFile" - {key, bucket} = req - logger.log key:key, bucket:bucket, "receiving request to delete file" - FileHandler.deleteFile bucket, key, (err)-> - if err? - logger.log err:err, key:key, bucket:bucket, "something went wrong deleting file" - res.send 500 - else - res.send 204 + deleteFile(req, res){ + metrics.inc("deleteFile"); + const {key, bucket} = req; + logger.log({key, bucket}, "receiving request to delete file"); + return FileHandler.deleteFile(bucket, key, function(err){ + if (err != null) { + logger.log({err, key, bucket}, "something went wrong deleting file"); + return res.send(500); + } else { + return res.send(204); + } + }); + }, - _get_range: (header) -> - parsed = parseRange(maxSizeInBytes, header) - if parsed == -1 or parsed == -2 or parsed.type != 'bytes' - null - else - range = parsed[0] - {start: range.start, end: range.end} + _get_range(header) { + const parsed = parseRange(maxSizeInBytes, header); + if ((parsed === -1) || (parsed === -2) || (parsed.type !== 'bytes')) { + return null; + } else { + const range = parsed[0]; + return {start: range.start, end: range.end}; + } + }, - directorySize: (req, res)-> - metrics.inc "projectSize" - {project_id, bucket} = req - logger.log project_id:project_id, bucket:bucket, "receiving request to project size" - FileHandler.getDirectorySize bucket, project_id, (err, size)-> - if err? - logger.log err: err, project_id: project_id, bucket: bucket, "error inserting file" - res.send 500 - else - res.json {'total bytes' : size} + directorySize(req, res){ + metrics.inc("projectSize"); + const {project_id, bucket} = req; + logger.log({project_id, bucket}, "receiving request to project size"); + return FileHandler.getDirectorySize(bucket, project_id, function(err, size){ + if (err != null) { + logger.log({err, project_id, bucket}, "error inserting file"); + return res.send(500); + } else { + return res.json({'total bytes' : size}); + } + }); + } +}); diff --git a/services/filestore/app/coffee/FileConverter.js b/services/filestore/app/coffee/FileConverter.js index 894b0e89fd..b35265c22b 100644 --- a/services/filestore/app/coffee/FileConverter.js +++ b/services/filestore/app/coffee/FileConverter.js @@ -1,62 +1,80 @@ -_ = require("underscore") -metrics = require("metrics-sharelatex") -logger = require("logger-sharelatex") -safe_exec = require("./SafeExec") -approvedFormats = ["png"] -Settings = require "settings-sharelatex" +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const _ = require("underscore"); +const metrics = require("metrics-sharelatex"); +const logger = require("logger-sharelatex"); +const safe_exec = require("./SafeExec"); +const approvedFormats = ["png"]; +const Settings = require("settings-sharelatex"); -fourtySeconds = 40 * 1000 +const fourtySeconds = 40 * 1000; -childProcessOpts = - killSignal: "SIGTERM" +const childProcessOpts = { + killSignal: "SIGTERM", timeout: fourtySeconds +}; -module.exports = +module.exports = { - convert: (sourcePath, requestedFormat, callback)-> - logger.log sourcePath:sourcePath, requestedFormat:requestedFormat, "converting file format" - timer = new metrics.Timer("imageConvert") - destPath = "#{sourcePath}.#{requestedFormat}" - sourcePath = "#{sourcePath}[0]" - if !_.include approvedFormats, requestedFormat - err = new Error("invalid format requested") - return callback err - width = "600x" - command = ["convert", "-define", "pdf:fit-page=#{width}", "-flatten", "-density", "300", sourcePath, destPath] - command = Settings.commands.convertCommandPrefix.concat(command) - safe_exec command, childProcessOpts, (err, stdout, stderr)-> - timer.done() - if err? - logger.err err:err, stderr:stderr, sourcePath:sourcePath, requestedFormat:requestedFormat, destPath:destPath, "something went wrong converting file" - else - logger.log sourcePath:sourcePath, requestedFormat:requestedFormat, destPath:destPath, "finished converting file" - callback(err, destPath) + convert(sourcePath, requestedFormat, callback){ + logger.log({sourcePath, requestedFormat}, "converting file format"); + const timer = new metrics.Timer("imageConvert"); + const destPath = `${sourcePath}.${requestedFormat}`; + sourcePath = `${sourcePath}[0]`; + if (!_.include(approvedFormats, requestedFormat)) { + const err = new Error("invalid format requested"); + return callback(err); + } + const width = "600x"; + let command = ["convert", "-define", `pdf:fit-page=${width}`, "-flatten", "-density", "300", sourcePath, destPath]; + command = Settings.commands.convertCommandPrefix.concat(command); + return safe_exec(command, childProcessOpts, function(err, stdout, stderr){ + timer.done(); + if (err != null) { + logger.err({err, stderr, sourcePath, requestedFormat, destPath}, "something went wrong converting file"); + } else { + logger.log({sourcePath, requestedFormat, destPath}, "finished converting file"); + } + return callback(err, destPath); + }); + }, - thumbnail: (sourcePath, callback)-> - destPath = "#{sourcePath}.png" - sourcePath = "#{sourcePath}[0]" - width = "260x" - command = ["convert", "-flatten", "-background", "white", "-density", "300", "-define", "pdf:fit-page=#{width}", sourcePath, "-resize", width, destPath] - logger.log sourcePath:sourcePath, destPath:destPath, command:command, "thumbnail convert file" - command = Settings.commands.convertCommandPrefix.concat(command) - safe_exec command, childProcessOpts, (err, stdout, stderr)-> - if err? - logger.err err:err, stderr:stderr, sourcePath:sourcePath, "something went wrong converting file to thumbnail" - else - logger.log sourcePath:sourcePath, destPath:destPath, "finished thumbnailing file" - callback(err, destPath) + thumbnail(sourcePath, callback){ + const destPath = `${sourcePath}.png`; + sourcePath = `${sourcePath}[0]`; + const width = "260x"; + let command = ["convert", "-flatten", "-background", "white", "-density", "300", "-define", `pdf:fit-page=${width}`, sourcePath, "-resize", width, destPath]; + logger.log({sourcePath, destPath, command}, "thumbnail convert file"); + command = Settings.commands.convertCommandPrefix.concat(command); + return safe_exec(command, childProcessOpts, function(err, stdout, stderr){ + if (err != null) { + logger.err({err, stderr, sourcePath}, "something went wrong converting file to thumbnail"); + } else { + logger.log({sourcePath, destPath}, "finished thumbnailing file"); + } + return callback(err, destPath); + }); + }, - preview: (sourcePath, callback)-> - logger.log sourcePath:sourcePath, "preview convert file" - destPath = "#{sourcePath}.png" - sourcePath = "#{sourcePath}[0]" - width = "548x" - command = ["convert", "-flatten", "-background", "white", "-density", "300", "-define", "pdf:fit-page=#{width}", sourcePath, "-resize", width, destPath] - command = Settings.commands.convertCommandPrefix.concat(command) - safe_exec command, childProcessOpts, (err, stdout, stderr)-> - if err? - logger.err err:err, stderr:stderr, sourcePath:sourcePath, destPath:destPath, "something went wrong converting file to preview" - else - logger.log sourcePath:sourcePath, destPath:destPath, "finished converting file to preview" - callback(err, destPath) + preview(sourcePath, callback){ + logger.log({sourcePath}, "preview convert file"); + const destPath = `${sourcePath}.png`; + sourcePath = `${sourcePath}[0]`; + const width = "548x"; + let command = ["convert", "-flatten", "-background", "white", "-density", "300", "-define", `pdf:fit-page=${width}`, sourcePath, "-resize", width, destPath]; + command = Settings.commands.convertCommandPrefix.concat(command); + return safe_exec(command, childProcessOpts, function(err, stdout, stderr){ + if (err != null) { + logger.err({err, stderr, sourcePath, destPath}, "something went wrong converting file to preview"); + } else { + logger.log({sourcePath, destPath}, "finished converting file to preview"); + } + return callback(err, destPath); + }); + } +}; diff --git a/services/filestore/app/coffee/FileHandler.js b/services/filestore/app/coffee/FileHandler.js index cb8d78a0fe..548e206a12 100644 --- a/services/filestore/app/coffee/FileHandler.js +++ b/services/filestore/app/coffee/FileHandler.js @@ -1,129 +1,169 @@ -settings = require("settings-sharelatex") -PersistorManager = require("./PersistorManager") -LocalFileWriter = require("./LocalFileWriter") -logger = require("logger-sharelatex") -FileConverter = require("./FileConverter") -KeyBuilder = require("./KeyBuilder") -async = require("async") -ImageOptimiser = require("./ImageOptimiser") -Errors = require('./Errors') +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let FileHandler; +const settings = require("settings-sharelatex"); +const PersistorManager = require("./PersistorManager"); +const LocalFileWriter = require("./LocalFileWriter"); +const logger = require("logger-sharelatex"); +const FileConverter = require("./FileConverter"); +const KeyBuilder = require("./KeyBuilder"); +const async = require("async"); +const ImageOptimiser = require("./ImageOptimiser"); +const Errors = require('./Errors'); -module.exports = FileHandler = +module.exports = (FileHandler = { - insertFile: (bucket, key, stream, callback)-> - convertedKey = KeyBuilder.getConvertedFolderKey key - PersistorManager.deleteDirectory bucket, convertedKey, (error) -> - return callback(error) if error? - PersistorManager.sendStream bucket, key, stream, callback + insertFile(bucket, key, stream, callback){ + const convertedKey = KeyBuilder.getConvertedFolderKey(key); + return PersistorManager.deleteDirectory(bucket, convertedKey, function(error) { + if (error != null) { return callback(error); } + return PersistorManager.sendStream(bucket, key, stream, callback); + }); + }, - deleteFile: (bucket, key, callback)-> - convertedKey = KeyBuilder.getConvertedFolderKey key - async.parallel [ - (done)-> PersistorManager.deleteFile bucket, key, done - (done)-> PersistorManager.deleteDirectory bucket, convertedKey, done - ], callback + deleteFile(bucket, key, callback){ + const convertedKey = KeyBuilder.getConvertedFolderKey(key); + return async.parallel([ + done => PersistorManager.deleteFile(bucket, key, done), + done => PersistorManager.deleteDirectory(bucket, convertedKey, done) + ], callback); + }, - getFile: (bucket, key, opts = {}, callback)-> - # In this call, opts can contain credentials - logger.log bucket:bucket, key:key, opts:@_scrubSecrets(opts), "getting file" - if !opts.format? and !opts.style? - @_getStandardFile bucket, key, opts, callback - else - @_getConvertedFile bucket, key, opts, callback + getFile(bucket, key, opts, callback){ + // In this call, opts can contain credentials + if (opts == null) { opts = {}; } + logger.log({bucket, key, opts:this._scrubSecrets(opts)}, "getting file"); + if ((opts.format == null) && (opts.style == null)) { + return this._getStandardFile(bucket, key, opts, callback); + } else { + return this._getConvertedFile(bucket, key, opts, callback); + } + }, - getFileSize: (bucket, key, callback) -> - PersistorManager.getFileSize(bucket, key, callback) + getFileSize(bucket, key, callback) { + return PersistorManager.getFileSize(bucket, key, callback); + }, - _getStandardFile: (bucket, key, opts, callback)-> - PersistorManager.getFileStream bucket, key, opts, (err, fileStream)-> - if err? and !(err instanceof Errors.NotFoundError) - logger.err bucket:bucket, key:key, opts:FileHandler._scrubSecrets(opts), "error getting fileStream" - callback err, fileStream + _getStandardFile(bucket, key, opts, callback){ + return PersistorManager.getFileStream(bucket, key, opts, function(err, fileStream){ + if ((err != null) && !(err instanceof Errors.NotFoundError)) { + logger.err({bucket, key, opts:FileHandler._scrubSecrets(opts)}, "error getting fileStream"); + } + return callback(err, fileStream); + }); + }, - _getConvertedFile: (bucket, key, opts, callback)-> - convertedKey = KeyBuilder.addCachingToKey key, opts - PersistorManager.checkIfFileExists bucket, convertedKey, (err, exists)=> - if err? - return callback err - if exists - PersistorManager.getFileStream bucket, convertedKey, opts, callback - else - @_getConvertedFileAndCache bucket, key, convertedKey, opts, callback + _getConvertedFile(bucket, key, opts, callback){ + const convertedKey = KeyBuilder.addCachingToKey(key, opts); + return PersistorManager.checkIfFileExists(bucket, convertedKey, (err, exists)=> { + if (err != null) { + return callback(err); + } + if (exists) { + return PersistorManager.getFileStream(bucket, convertedKey, opts, callback); + } else { + return this._getConvertedFileAndCache(bucket, key, convertedKey, opts, callback); + } + }); + }, - _getConvertedFileAndCache: (bucket, key, convertedKey, opts, callback)-> - convertedFsPath = "" - originalFsPath = "" - async.series [ - (cb) => - @_convertFile bucket, key, opts, (err, fileSystemPath, originalFsPath) -> - convertedFsPath = fileSystemPath - originalFsPath = originalFsPath - cb err - (cb)-> - ImageOptimiser.compressPng convertedFsPath, cb - (cb)-> - PersistorManager.sendFile bucket, convertedKey, convertedFsPath, cb - ], (err)-> - if err? - LocalFileWriter.deleteFile convertedFsPath, -> - LocalFileWriter.deleteFile originalFsPath, -> - return callback(err) - # Send back the converted file from the local copy to avoid problems - # with the file not being present in S3 yet. As described in the - # documentation below, we have already made a 'HEAD' request in - # checkIfFileExists so we only have "eventual consistency" if we try - # to stream it from S3 here. This was a cause of many 403 errors. - # - # "Amazon S3 provides read-after-write consistency for PUTS of new - # objects in your S3 bucket in all regions with one caveat. The - # caveat is that if you make a HEAD or GET request to the key name - # (to find if the object exists) before creating the object, Amazon - # S3 provides eventual consistency for read-after-write."" - # https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel - LocalFileWriter.getStream convertedFsPath, (err, readStream) -> - return callback(err) if err? - readStream.on 'end', () -> - logger.log {convertedFsPath: convertedFsPath}, "deleting temporary file" - LocalFileWriter.deleteFile convertedFsPath, -> - callback(null, readStream) + _getConvertedFileAndCache(bucket, key, convertedKey, opts, callback){ + let convertedFsPath = ""; + const originalFsPath = ""; + return async.series([ + cb => { + return this._convertFile(bucket, key, opts, function(err, fileSystemPath, originalFsPath) { + convertedFsPath = fileSystemPath; + originalFsPath = originalFsPath; + return cb(err); + }); + }, + cb => ImageOptimiser.compressPng(convertedFsPath, cb), + cb => PersistorManager.sendFile(bucket, convertedKey, convertedFsPath, cb) + ], function(err){ + if (err != null) { + LocalFileWriter.deleteFile(convertedFsPath, function() {}); + LocalFileWriter.deleteFile(originalFsPath, function() {}); + return callback(err); + } + // Send back the converted file from the local copy to avoid problems + // with the file not being present in S3 yet. As described in the + // documentation below, we have already made a 'HEAD' request in + // checkIfFileExists so we only have "eventual consistency" if we try + // to stream it from S3 here. This was a cause of many 403 errors. + // + // "Amazon S3 provides read-after-write consistency for PUTS of new + // objects in your S3 bucket in all regions with one caveat. The + // caveat is that if you make a HEAD or GET request to the key name + // (to find if the object exists) before creating the object, Amazon + // S3 provides eventual consistency for read-after-write."" + // https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel + return LocalFileWriter.getStream(convertedFsPath, function(err, readStream) { + if (err != null) { return callback(err); } + readStream.on('end', function() { + logger.log({convertedFsPath}, "deleting temporary file"); + return LocalFileWriter.deleteFile(convertedFsPath, function() {}); + }); + return callback(null, readStream); + }); + }); + }, - _convertFile: (bucket, originalKey, opts, callback)-> - @_writeS3FileToDisk bucket, originalKey, opts, (err, originalFsPath)-> - if err? - return callback(err) - done = (err, destPath)-> - if err? - logger.err err:err, bucket:bucket, originalKey:originalKey, opts:FileHandler._scrubSecrets(opts), "error converting file" - return callback(err) - LocalFileWriter.deleteFile originalFsPath, -> - callback(err, destPath, originalFsPath) + _convertFile(bucket, originalKey, opts, callback){ + return this._writeS3FileToDisk(bucket, originalKey, opts, function(err, originalFsPath){ + if (err != null) { + return callback(err); + } + const done = function(err, destPath){ + if (err != null) { + logger.err({err, bucket, originalKey, opts:FileHandler._scrubSecrets(opts)}, "error converting file"); + return callback(err); + } + LocalFileWriter.deleteFile(originalFsPath, function() {}); + return callback(err, destPath, originalFsPath); + }; - logger.log opts:opts, "converting file depending on opts" + logger.log({opts}, "converting file depending on opts"); - if opts.format? - FileConverter.convert originalFsPath, opts.format, done - else if opts.style == "thumbnail" - FileConverter.thumbnail originalFsPath, done - else if opts.style == "preview" - FileConverter.preview originalFsPath, done - else - return callback(new Error("should have specified opts to convert file with #{JSON.stringify(opts)}")) + if (opts.format != null) { + return FileConverter.convert(originalFsPath, opts.format, done); + } else if (opts.style === "thumbnail") { + return FileConverter.thumbnail(originalFsPath, done); + } else if (opts.style === "preview") { + return FileConverter.preview(originalFsPath, done); + } else { + return callback(new Error(`should have specified opts to convert file with ${JSON.stringify(opts)}`)); + } + }); + }, - _writeS3FileToDisk: (bucket, key, opts, callback)-> - PersistorManager.getFileStream bucket, key, opts, (err, fileStream)-> - if err? - return callback(err) - LocalFileWriter.writeStream fileStream, key, callback + _writeS3FileToDisk(bucket, key, opts, callback){ + return PersistorManager.getFileStream(bucket, key, opts, function(err, fileStream){ + if (err != null) { + return callback(err); + } + return LocalFileWriter.writeStream(fileStream, key, callback); + }); + }, - getDirectorySize: (bucket, project_id, callback)-> - logger.log bucket:bucket, project_id:project_id, "getting project size" - PersistorManager.directorySize bucket, project_id, (err, size)-> - if err? - logger.err bucket:bucket, project_id:project_id, "error getting size" - callback err, size + getDirectorySize(bucket, project_id, callback){ + logger.log({bucket, project_id}, "getting project size"); + return PersistorManager.directorySize(bucket, project_id, function(err, size){ + if (err != null) { + logger.err({bucket, project_id}, "error getting size"); + } + return callback(err, size); + }); + }, - _scrubSecrets: (opts)-> - safe = Object.assign {}, opts - delete safe.credentials - safe + _scrubSecrets(opts){ + const safe = Object.assign({}, opts); + delete safe.credentials; + return safe; + } +}); diff --git a/services/filestore/app/coffee/HealthCheckController.js b/services/filestore/app/coffee/HealthCheckController.js index 643e15adb3..4f49f4a19a 100644 --- a/services/filestore/app/coffee/HealthCheckController.js +++ b/services/filestore/app/coffee/HealthCheckController.js @@ -1,57 +1,74 @@ -fs = require("fs-extra") -path = require("path") -async = require("async") -fileConverter = require("./FileConverter") -keyBuilder = require("./KeyBuilder") -fileController = require("./FileController") -logger = require('logger-sharelatex') -settings = require("settings-sharelatex") -streamBuffers = require("stream-buffers") -_ = require('underscore') +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const fs = require("fs-extra"); +const path = require("path"); +const async = require("async"); +const fileConverter = require("./FileConverter"); +const keyBuilder = require("./KeyBuilder"); +const fileController = require("./FileController"); +const logger = require('logger-sharelatex'); +const settings = require("settings-sharelatex"); +const streamBuffers = require("stream-buffers"); +const _ = require('underscore'); -checkCanStoreFiles = (callback)-> - callback = _.once(callback) - req = {params:{}, query:{}, headers:{}} - req.params.project_id = settings.health_check.project_id - req.params.file_id = settings.health_check.file_id - myWritableStreamBuffer = new streamBuffers.WritableStreamBuffer(initialSize: 100) - res = { - send: (code) -> - if code != 200 - callback(new Error("non-200 code from getFile: #{code}")) +const checkCanStoreFiles = function(callback){ + callback = _.once(callback); + const req = {params:{}, query:{}, headers:{}}; + req.params.project_id = settings.health_check.project_id; + req.params.file_id = settings.health_check.file_id; + const myWritableStreamBuffer = new streamBuffers.WritableStreamBuffer({initialSize: 100}); + const res = { + send(code) { + if (code !== 200) { + return callback(new Error(`non-200 code from getFile: ${code}`)); + } + } + }; + myWritableStreamBuffer.send = res.send; + return keyBuilder.userFileKey(req, res, function() { + fileController.getFile(req, myWritableStreamBuffer); + return myWritableStreamBuffer.on("close", function() { + if (myWritableStreamBuffer.size() > 0) { + return callback(); + } else { + const err = "no data in write stream buffer for health check"; + logger.err({err,}, "error performing health check"); + return callback(err); + } + }); + }); +}; + +const checkFileConvert = function(callback){ + if (!settings.enableConversions) { + return callback(); } - myWritableStreamBuffer.send = res.send - keyBuilder.userFileKey req, res, -> - fileController.getFile req, myWritableStreamBuffer - myWritableStreamBuffer.on "close", -> - if myWritableStreamBuffer.size() > 0 - callback() - else - err = "no data in write stream buffer for health check" - logger.err {err,}, "error performing health check" - callback(err) - -checkFileConvert = (callback)-> - if !settings.enableConversions - return callback() - imgPath = path.join(settings.path.uploadFolder, "/tiny.pdf") - async.waterfall [ - (cb)-> - fs.copy("./tiny.pdf", imgPath, cb) - (cb)-> fileConverter.thumbnail imgPath, cb - (resultPath, cb)-> fs.unlink resultPath, cb - (cb)-> fs.unlink imgPath, cb - ], callback + const imgPath = path.join(settings.path.uploadFolder, "/tiny.pdf"); + return async.waterfall([ + cb => fs.copy("./tiny.pdf", imgPath, cb), + cb => fileConverter.thumbnail(imgPath, cb), + (resultPath, cb) => fs.unlink(resultPath, cb), + cb => fs.unlink(imgPath, cb) + ], callback); +}; -module.exports = +module.exports = { - check: (req, res) -> - logger.log {}, "performing health check" - async.parallel [checkFileConvert, checkCanStoreFiles], (err)-> - if err? - logger.err err:err, "Health check: error running" - res.send 500 - else - res.send 200 + check(req, res) { + logger.log({}, "performing health check"); + return async.parallel([checkFileConvert, checkCanStoreFiles], function(err){ + if (err != null) { + logger.err({err}, "Health check: error running"); + return res.send(500); + } else { + return res.send(200); + } + }); + } +}; diff --git a/services/filestore/app/coffee/ImageOptimiser.js b/services/filestore/app/coffee/ImageOptimiser.js index 4c4a353f21..570730618d 100644 --- a/services/filestore/app/coffee/ImageOptimiser.js +++ b/services/filestore/app/coffee/ImageOptimiser.js @@ -1,25 +1,39 @@ -exec = require('child_process').exec -logger = require("logger-sharelatex") -Settings = require "settings-sharelatex" +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const { + exec +} = require('child_process'); +const logger = require("logger-sharelatex"); +const Settings = require("settings-sharelatex"); -module.exports = +module.exports = { - compressPng: (localPath, callback)-> - startTime = new Date() - logger.log localPath:localPath, "optimising png path" - args = "optipng #{localPath}" - opts = - timeout: 30 * 1000 + compressPng(localPath, callback){ + const startTime = new Date(); + logger.log({localPath}, "optimising png path"); + const args = `optipng ${localPath}`; + const opts = { + timeout: 30 * 1000, killSignal: "SIGKILL" - if !Settings.enableConversions - error = new Error("Image conversions are disabled") - return callback(error) - exec args, opts,(err, stdout, stderr)-> - if err? and err.signal == 'SIGKILL' - logger.warn {err: err, stderr: stderr, localPath: localPath}, "optimiser timeout reached" - err = null - else if err? - logger.err err:err, stderr:stderr, localPath:localPath, "something went wrong converting compressPng" - else - logger.log localPath:localPath, "finished compressPng file" - callback(err) + }; + if (!Settings.enableConversions) { + const error = new Error("Image conversions are disabled"); + return callback(error); + } + return exec(args, opts,function(err, stdout, stderr){ + if ((err != null) && (err.signal === 'SIGKILL')) { + logger.warn({err, stderr, localPath}, "optimiser timeout reached"); + err = null; + } else if (err != null) { + logger.err({err, stderr, localPath}, "something went wrong converting compressPng"); + } else { + logger.log({localPath}, "finished compressPng file"); + } + return callback(err); + }); + } +}; diff --git a/services/filestore/app/coffee/KeyBuilder.js b/services/filestore/app/coffee/KeyBuilder.js index 18f1d96952..f5c8b7d5da 100644 --- a/services/filestore/app/coffee/KeyBuilder.js +++ b/services/filestore/app/coffee/KeyBuilder.js @@ -1,50 +1,68 @@ -settings = require("settings-sharelatex") +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const settings = require("settings-sharelatex"); -module.exports = +module.exports = { - getConvertedFolderKey: (key)-> - key = "#{key}-converted-cache/" + getConvertedFolderKey(key){ + return key = `${key}-converted-cache/`; + }, - addCachingToKey: (key, opts)-> - key = @getConvertedFolderKey(key) - if opts.format? and !opts.style? - key = "#{key}format-#{opts.format}" - if opts.style? and !opts.format? - key = "#{key}style-#{opts.style}" - if opts.style? and opts.format? - key = "#{key}format-#{opts.format}-style-#{opts.style}" - return key + addCachingToKey(key, opts){ + key = this.getConvertedFolderKey(key); + if ((opts.format != null) && (opts.style == null)) { + key = `${key}format-${opts.format}`; + } + if ((opts.style != null) && (opts.format == null)) { + key = `${key}style-${opts.style}`; + } + if ((opts.style != null) && (opts.format != null)) { + key = `${key}format-${opts.format}-style-${opts.style}`; + } + return key; + }, - userFileKey: (req, res, next)-> - {project_id, file_id} = req.params - req.key = "#{project_id}/#{file_id}" - req.bucket = settings.filestore.stores.user_files - next() + userFileKey(req, res, next){ + const {project_id, file_id} = req.params; + req.key = `${project_id}/${file_id}`; + req.bucket = settings.filestore.stores.user_files; + return next(); + }, - publicFileKey: (req, res, next)-> - {project_id, public_file_id} = req.params - if not settings.filestore.stores.public_files? - res.status(501).send("public files not available") - else - req.key = "#{project_id}/#{public_file_id}" - req.bucket = settings.filestore.stores.public_files - next() + publicFileKey(req, res, next){ + const {project_id, public_file_id} = req.params; + if ((settings.filestore.stores.public_files == null)) { + return res.status(501).send("public files not available"); + } else { + req.key = `${project_id}/${public_file_id}`; + req.bucket = settings.filestore.stores.public_files; + return next(); + } + }, - templateFileKey: (req, res, next)-> - {template_id, format, version, sub_type} = req.params - req.key = "#{template_id}/v/#{version}/#{format}" - if sub_type? - req.key = "#{req.key}/#{sub_type}" - req.bucket = settings.filestore.stores.template_files - req.version = version - opts = req.query - next() + templateFileKey(req, res, next){ + const {template_id, format, version, sub_type} = req.params; + req.key = `${template_id}/v/${version}/${format}`; + if (sub_type != null) { + req.key = `${req.key}/${sub_type}`; + } + req.bucket = settings.filestore.stores.template_files; + req.version = version; + const opts = req.query; + return next(); + }, - publicProjectKey: (req, res, next)-> - {project_id} = req.params - req.project_id = project_id - req.bucket = settings.filestore.stores.user_files - next() + publicProjectKey(req, res, next){ + const {project_id} = req.params; + req.project_id = project_id; + req.bucket = settings.filestore.stores.user_files; + return next(); + } +}; diff --git a/services/filestore/app/coffee/LocalFileWriter.js b/services/filestore/app/coffee/LocalFileWriter.js index 7046da15e7..a6457de526 100644 --- a/services/filestore/app/coffee/LocalFileWriter.js +++ b/services/filestore/app/coffee/LocalFileWriter.js @@ -1,56 +1,76 @@ -fs = require("fs") -uuid = require('node-uuid') -path = require("path") -_ = require("underscore") -logger = require("logger-sharelatex") -metrics = require("metrics-sharelatex") -Settings = require("settings-sharelatex") -Errors = require "./Errors" +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const fs = require("fs"); +const uuid = require('node-uuid'); +const path = require("path"); +const _ = require("underscore"); +const logger = require("logger-sharelatex"); +const metrics = require("metrics-sharelatex"); +const Settings = require("settings-sharelatex"); +const Errors = require("./Errors"); -module.exports = +module.exports = { - writeStream: (stream, key, callback)-> - timer = new metrics.Timer("writingFile") - callback = _.once callback - fsPath = @_getPath(key) - logger.log fsPath:fsPath, "writing file locally" - writeStream = fs.createWriteStream(fsPath) - writeStream.on "finish", -> - timer.done() - logger.log fsPath:fsPath, "finished writing file locally" - callback(null, fsPath) - writeStream.on "error", (err)-> - logger.err err:err, fsPath:fsPath, "problem writing file locally, with write stream" - callback err - stream.on "error", (err)-> - logger.log err:err, fsPath:fsPath, "problem writing file locally, with read stream" - callback err - stream.pipe writeStream + writeStream(stream, key, callback){ + const timer = new metrics.Timer("writingFile"); + callback = _.once(callback); + const fsPath = this._getPath(key); + logger.log({fsPath}, "writing file locally"); + const writeStream = fs.createWriteStream(fsPath); + writeStream.on("finish", function() { + timer.done(); + logger.log({fsPath}, "finished writing file locally"); + return callback(null, fsPath); + }); + writeStream.on("error", function(err){ + logger.err({err, fsPath}, "problem writing file locally, with write stream"); + return callback(err); + }); + stream.on("error", function(err){ + logger.log({err, fsPath}, "problem writing file locally, with read stream"); + return callback(err); + }); + return stream.pipe(writeStream); + }, - getStream: (fsPath, _callback = (err, res)->) -> - callback = _.once _callback - timer = new metrics.Timer("readingFile") - logger.log fsPath:fsPath, "reading file locally" - readStream = fs.createReadStream(fsPath) - readStream.on "end", -> - timer.done() - logger.log fsPath:fsPath, "finished reading file locally" - readStream.on "error", (err)-> - logger.err err:err, fsPath:fsPath, "problem reading file locally, with read stream" - if err.code == 'ENOENT' - callback new Errors.NotFoundError(err.message), null - else - callback err - callback null, readStream + getStream(fsPath, _callback) { + if (_callback == null) { _callback = function(err, res){}; } + const callback = _.once(_callback); + const timer = new metrics.Timer("readingFile"); + logger.log({fsPath}, "reading file locally"); + const readStream = fs.createReadStream(fsPath); + readStream.on("end", function() { + timer.done(); + return logger.log({fsPath}, "finished reading file locally"); + }); + readStream.on("error", function(err){ + logger.err({err, fsPath}, "problem reading file locally, with read stream"); + if (err.code === 'ENOENT') { + return callback(new Errors.NotFoundError(err.message), null); + } else { + return callback(err); + } + }); + return callback(null, readStream); + }, - deleteFile: (fsPath, callback)-> - if !fsPath? or fsPath == "" - return callback() - logger.log fsPath:fsPath, "removing local temp file" - fs.unlink fsPath, callback + deleteFile(fsPath, callback){ + if ((fsPath == null) || (fsPath === "")) { + return callback(); + } + logger.log({fsPath}, "removing local temp file"); + return fs.unlink(fsPath, callback); + }, - _getPath : (key)-> - if !key? - key = uuid.v1() - key = key.replace(/\//g,"-") - path.join(Settings.path.uploadFolder, key) + _getPath(key){ + if ((key == null)) { + key = uuid.v1(); + } + key = key.replace(/\//g,"-"); + return path.join(Settings.path.uploadFolder, key); + } +}; diff --git a/services/filestore/app/coffee/PersistorManager.js b/services/filestore/app/coffee/PersistorManager.js index 85c9dd83cd..993a874d6a 100644 --- a/services/filestore/app/coffee/PersistorManager.js +++ b/services/filestore/app/coffee/PersistorManager.js @@ -1,16 +1,28 @@ -settings = require("settings-sharelatex") -logger = require("logger-sharelatex") +/* + * decaffeinate suggestions: + * DS103: Rewrite code to no longer use __guard__ + * DS205: Consider reworking code to avoid use of IIFEs + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const settings = require("settings-sharelatex"); +const logger = require("logger-sharelatex"); -# assume s3 if none specified -settings?.filestore?.backend ||= "s3" +// assume s3 if none specified +__guard__(settings != null ? settings.filestore : undefined, x => x.backend || (settings.filestore.backend = "s3")); -logger.log backend:settings?.filestore?.backend, "Loading backend" -module.exports = switch settings?.filestore?.backend - when "aws-sdk" - require "./AWSSDKPersistorManager" - when "s3" - require("./S3PersistorManager") - when "fs" - require("./FSPersistorManager") - else - throw new Error( "Unknown filestore backend: #{settings.filestore.backend}" ) +logger.log({backend:__guard__(settings != null ? settings.filestore : undefined, x1 => x1.backend)}, "Loading backend"); +module.exports = (() => { switch (__guard__(settings != null ? settings.filestore : undefined, x2 => x2.backend)) { + case "aws-sdk": + return require("./AWSSDKPersistorManager"); + case "s3": + return require("./S3PersistorManager"); + case "fs": + return require("./FSPersistorManager"); + default: + throw new Error( `Unknown filestore backend: ${settings.filestore.backend}` ); +} })(); + +function __guard__(value, transform) { + return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; +} \ No newline at end of file diff --git a/services/filestore/app/coffee/S3PersistorManager.js b/services/filestore/app/coffee/S3PersistorManager.js index 89522a4643..a8edbc6844 100644 --- a/services/filestore/app/coffee/S3PersistorManager.js +++ b/services/filestore/app/coffee/S3PersistorManager.js @@ -1,238 +1,290 @@ -# This module is the one which is used in production. It needs to be migrated -# to use aws-sdk throughout, see the comments in AWSSDKPersistorManager for -# details. The knox library is unmaintained and has bugs. +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +// This module is the one which is used in production. It needs to be migrated +// to use aws-sdk throughout, see the comments in AWSSDKPersistorManager for +// details. The knox library is unmaintained and has bugs. -http = require('http') -http.globalAgent.maxSockets = 300 -https = require('https') -https.globalAgent.maxSockets = 300 -settings = require("settings-sharelatex") -request = require("request") -logger = require("logger-sharelatex") -metrics = require("metrics-sharelatex") -fs = require("fs") -knox = require("knox") -path = require("path") -LocalFileWriter = require("./LocalFileWriter") -Errors = require("./Errors") -_ = require("underscore") -awsS3 = require "aws-sdk/clients/s3" -URL = require('url') +const http = require('http'); +http.globalAgent.maxSockets = 300; +const https = require('https'); +https.globalAgent.maxSockets = 300; +const settings = require("settings-sharelatex"); +const request = require("request"); +const logger = require("logger-sharelatex"); +const metrics = require("metrics-sharelatex"); +const fs = require("fs"); +const knox = require("knox"); +const path = require("path"); +const LocalFileWriter = require("./LocalFileWriter"); +const Errors = require("./Errors"); +const _ = require("underscore"); +const awsS3 = require("aws-sdk/clients/s3"); +const URL = require('url'); -thirtySeconds = 30 * 1000 +const thirtySeconds = 30 * 1000; -buildDefaultOptions = (bucketName, method, key)-> - if settings.filestore.s3.endpoint - endpoint = "#{settings.filestore.s3.endpoint}/#{bucketName}" - else - endpoint = "https://#{bucketName}.s3.amazonaws.com" +const buildDefaultOptions = function(bucketName, method, key){ + let endpoint; + if (settings.filestore.s3.endpoint) { + endpoint = `${settings.filestore.s3.endpoint}/${bucketName}`; + } else { + endpoint = `https://${bucketName}.s3.amazonaws.com`; + } return { - aws: - key: settings.filestore.s3.key - secret: settings.filestore.s3.secret + aws: { + key: settings.filestore.s3.key, + secret: settings.filestore.s3.secret, bucket: bucketName - method: method - timeout: thirtySeconds - uri:"#{endpoint}/#{key}" + }, + method, + timeout: thirtySeconds, + uri:`${endpoint}/${key}` + }; +}; + +const getS3Options = function(credentials) { + const options = { + credentials: { + accessKeyId: credentials.auth_key, + secretAccessKey: credentials.auth_secret + } + }; + + if (settings.filestore.s3.endpoint) { + const endpoint = URL.parse(settings.filestore.s3.endpoint); + options.endpoint = settings.filestore.s3.endpoint; + options.sslEnabled = endpoint.protocol === 'https'; } -getS3Options = (credentials) -> - options = - credentials: - accessKeyId: credentials.auth_key - secretAccessKey: credentials.auth_secret + return options; +}; - if settings.filestore.s3.endpoint - endpoint = URL.parse(settings.filestore.s3.endpoint) - options.endpoint = settings.filestore.s3.endpoint - options.sslEnabled = endpoint.protocol == 'https' - - return options - -defaultS3Client = new awsS3(getS3Options({ +const defaultS3Client = new awsS3(getS3Options({ auth_key: settings.filestore.s3.key, auth_secret: settings.filestore.s3.secret -})) +})); -getS3Client = (credentials) -> - if credentials? - return new awsS3(getS3Options(credentials)) - else - return defaultS3Client +const getS3Client = function(credentials) { + if (credentials != null) { + return new awsS3(getS3Options(credentials)); + } else { + return defaultS3Client; + } +}; -getKnoxClient = (bucketName) => - options = - key: settings.filestore.s3.key - secret: settings.filestore.s3.secret +const getKnoxClient = bucketName => { + const options = { + key: settings.filestore.s3.key, + secret: settings.filestore.s3.secret, bucket: bucketName - if settings.filestore.s3.endpoint - endpoint = URL.parse(settings.filestore.s3.endpoint) - options.endpoint = endpoint.hostname - options.port = endpoint.port - return knox.createClient(options) + }; + if (settings.filestore.s3.endpoint) { + const endpoint = URL.parse(settings.filestore.s3.endpoint); + options.endpoint = endpoint.hostname; + options.port = endpoint.port; + } + return knox.createClient(options); +}; -module.exports = +module.exports = { - sendFile: (bucketName, key, fsPath, callback)-> - s3Client = getKnoxClient(bucketName) - uploaded = 0 - putEventEmiter = s3Client.putFile fsPath, key, (err, res)-> - metrics.count 's3.egress', uploaded - if err? - logger.err err:err, bucketName:bucketName, key:key, fsPath:fsPath,"something went wrong uploading file to s3" - return callback(err) - if !res? - logger.err err:err, res:res, bucketName:bucketName, key:key, fsPath:fsPath, "no response from s3 put file" - return callback("no response from put file") - if res.statusCode != 200 - logger.err bucketName:bucketName, key:key, fsPath:fsPath, "non 200 response from s3 putting file" - return callback("non 200 response from s3 on put file") - logger.log res:res, bucketName:bucketName, key:key, fsPath:fsPath,"file uploaded to s3" - callback(err) - putEventEmiter.on "error", (err)-> - logger.err err:err, bucketName:bucketName, key:key, fsPath:fsPath, "error emmited on put of file" - callback err - putEventEmiter.on "progress", (progress)-> - uploaded = progress.written + sendFile(bucketName, key, fsPath, callback){ + const s3Client = getKnoxClient(bucketName); + let uploaded = 0; + const putEventEmiter = s3Client.putFile(fsPath, key, function(err, res){ + metrics.count('s3.egress', uploaded); + if (err != null) { + logger.err({err, bucketName, key, fsPath},"something went wrong uploading file to s3"); + return callback(err); + } + if ((res == null)) { + logger.err({err, res, bucketName, key, fsPath}, "no response from s3 put file"); + return callback("no response from put file"); + } + if (res.statusCode !== 200) { + logger.err({bucketName, key, fsPath}, "non 200 response from s3 putting file"); + return callback("non 200 response from s3 on put file"); + } + logger.log({res, bucketName, key, fsPath},"file uploaded to s3"); + return callback(err); + }); + putEventEmiter.on("error", function(err){ + logger.err({err, bucketName, key, fsPath}, "error emmited on put of file"); + return callback(err); + }); + return putEventEmiter.on("progress", progress => uploaded = progress.written); + }, - sendStream: (bucketName, key, readStream, callback)-> - logger.log bucketName:bucketName, key:key, "sending file to s3" - readStream.on "error", (err)-> - logger.err bucketName:bucketName, key:key, "error on stream to send to s3" - LocalFileWriter.writeStream readStream, null, (err, fsPath)=> - if err? - logger.err bucketName:bucketName, key:key, fsPath:fsPath, err:err, "something went wrong writing stream to disk" - return callback(err) - @sendFile bucketName, key, fsPath, (err) -> - # delete the temporary file created above and return the original error - LocalFileWriter.deleteFile fsPath, () -> - callback(err) + sendStream(bucketName, key, readStream, callback){ + logger.log({bucketName, key}, "sending file to s3"); + readStream.on("error", err => logger.err({bucketName, key}, "error on stream to send to s3")); + return LocalFileWriter.writeStream(readStream, null, (err, fsPath)=> { + if (err != null) { + logger.err({bucketName, key, fsPath, err}, "something went wrong writing stream to disk"); + return callback(err); + } + return this.sendFile(bucketName, key, fsPath, err => // delete the temporary file created above and return the original error + LocalFileWriter.deleteFile(fsPath, () => callback(err))); + }); + }, - # opts may be {start: Number, end: Number} - getFileStream: (bucketName, key, opts, callback = (err, res)->)-> - opts = opts || {} - callback = _.once(callback) - logger.log bucketName:bucketName, key:key, "getting file from s3" + // opts may be {start: Number, end: Number} + getFileStream(bucketName, key, opts, callback){ + if (callback == null) { callback = function(err, res){}; } + opts = opts || {}; + callback = _.once(callback); + logger.log({bucketName, key}, "getting file from s3"); - s3 = getS3Client(opts.credentials) - s3Params = { - Bucket: bucketName + const s3 = getS3Client(opts.credentials); + const s3Params = { + Bucket: bucketName, Key: key + }; + if ((opts.start != null) && (opts.end != null)) { + s3Params['Range'] = `bytes=${opts.start}-${opts.end}`; } - if opts.start? and opts.end? - s3Params['Range'] = "bytes=#{opts.start}-#{opts.end}" - s3Request = s3.getObject(s3Params) + const s3Request = s3.getObject(s3Params); - s3Request.on 'httpHeaders', (statusCode, headers, response, statusMessage) => - if statusCode in [403, 404] - # S3 returns a 403 instead of a 404 when the user doesn't have - # permission to list the bucket contents. - logger.log({ bucketName: bucketName, key: key }, "file not found in s3") - return callback(new Errors.NotFoundError("File not found in S3: #{bucketName}:#{key}"), null) - if statusCode not in [200, 206] - logger.log({bucketName: bucketName, key: key }, "error getting file from s3: #{statusCode}") - return callback(new Error("Got non-200 response from S3: #{statusCode} #{statusMessage}"), null) - stream = response.httpResponse.createUnbufferedStream() - stream.on 'data', (data) -> - metrics.count 's3.ingress', data.byteLength + s3Request.on('httpHeaders', (statusCode, headers, response, statusMessage) => { + if ([403, 404].includes(statusCode)) { + // S3 returns a 403 instead of a 404 when the user doesn't have + // permission to list the bucket contents. + logger.log({ bucketName, key }, "file not found in s3"); + return callback(new Errors.NotFoundError(`File not found in S3: ${bucketName}:${key}`), null); + } + if (![200, 206].includes(statusCode)) { + logger.log({bucketName, key }, `error getting file from s3: ${statusCode}`); + return callback(new Error(`Got non-200 response from S3: ${statusCode} ${statusMessage}`), null); + } + const stream = response.httpResponse.createUnbufferedStream(); + stream.on('data', data => metrics.count('s3.ingress', data.byteLength)); - callback(null, stream) + return callback(null, stream); + }); - s3Request.on 'error', (err) => - logger.err({ err: err, bucketName: bucketName, key: key }, "error getting file stream from s3") - callback(err) + s3Request.on('error', err => { + logger.err({ err, bucketName, key }, "error getting file stream from s3"); + return callback(err); + }); - s3Request.send() + return s3Request.send(); + }, - getFileSize: (bucketName, key, callback) -> - logger.log({ bucketName: bucketName, key: key }, "getting file size from S3") - s3 = getS3Client() - s3.headObject { Bucket: bucketName, Key: key }, (err, data) -> - if err? - if err.statusCode in [403, 404] - # S3 returns a 403 instead of a 404 when the user doesn't have - # permission to list the bucket contents. + getFileSize(bucketName, key, callback) { + logger.log({ bucketName, key }, "getting file size from S3"); + const s3 = getS3Client(); + return s3.headObject({ Bucket: bucketName, Key: key }, function(err, data) { + if (err != null) { + if ([403, 404].includes(err.statusCode)) { + // S3 returns a 403 instead of a 404 when the user doesn't have + // permission to list the bucket contents. logger.log({ - bucketName: bucketName, - key: key - }, "file not found in s3") + bucketName, + key + }, "file not found in s3"); callback( - new Errors.NotFoundError("File not found in S3: #{bucketName}:#{key}") - ) - else + new Errors.NotFoundError(`File not found in S3: ${bucketName}:${key}`) + ); + } else { logger.err({ - bucketName: bucketName, - key: key, - err: err - }, "error performing S3 HeadObject") - callback(err) - return - callback(null, data.ContentLength) + bucketName, + key, + err + }, "error performing S3 HeadObject"); + callback(err); + } + return; + } + return callback(null, data.ContentLength); + }); + }, - copyFile: (bucketName, sourceKey, destKey, callback)-> - logger.log bucketName:bucketName, sourceKey:sourceKey, destKey: destKey, "copying file in s3" - source = bucketName + '/' + sourceKey - # use the AWS SDK instead of knox due to problems with error handling (https://github.com/Automattic/knox/issues/114) - s3 = getS3Client() - s3.copyObject {Bucket: bucketName, Key: destKey, CopySource: source}, (err) -> - if err? - if err.code is 'NoSuchKey' - logger.err bucketName:bucketName, sourceKey:sourceKey, "original file not found in s3 when copying" - callback(new Errors.NotFoundError("original file not found in S3 when copying")) - else - logger.err err:err, bucketName:bucketName, sourceKey:sourceKey, destKey:destKey, "something went wrong copying file in aws" - callback(err) - else - callback() + copyFile(bucketName, sourceKey, destKey, callback){ + logger.log({bucketName, sourceKey, destKey}, "copying file in s3"); + const source = bucketName + '/' + sourceKey; + // use the AWS SDK instead of knox due to problems with error handling (https://github.com/Automattic/knox/issues/114) + const s3 = getS3Client(); + return s3.copyObject({Bucket: bucketName, Key: destKey, CopySource: source}, function(err) { + if (err != null) { + if (err.code === 'NoSuchKey') { + logger.err({bucketName, sourceKey}, "original file not found in s3 when copying"); + return callback(new Errors.NotFoundError("original file not found in S3 when copying")); + } else { + logger.err({err, bucketName, sourceKey, destKey}, "something went wrong copying file in aws"); + return callback(err); + } + } else { + return callback(); + } + }); + }, - deleteFile: (bucketName, key, callback)-> - logger.log bucketName:bucketName, key:key, "delete file in s3" - options = buildDefaultOptions(bucketName, "delete", key) - request options, (err, res)-> - if err? - logger.err err:err, res:res, bucketName:bucketName, key:key, "something went wrong deleting file in aws" - callback(err) + deleteFile(bucketName, key, callback){ + logger.log({bucketName, key}, "delete file in s3"); + const options = buildDefaultOptions(bucketName, "delete", key); + return request(options, function(err, res){ + if (err != null) { + logger.err({err, res, bucketName, key}, "something went wrong deleting file in aws"); + } + return callback(err); + }); + }, - deleteDirectory: (bucketName, key, _callback)-> - # deleteMultiple can call the callback multiple times so protect against this. - callback = (args...) -> - _callback(args...) - _callback = () -> + deleteDirectory(bucketName, key, _callback){ + // deleteMultiple can call the callback multiple times so protect against this. + const callback = function(...args) { + _callback(...Array.from(args || [])); + return _callback = function() {}; + }; - logger.log key: key, bucketName: bucketName, "deleting directory" - s3Client = getKnoxClient(bucketName) - s3Client.list prefix:key, (err, data)-> - if err? - logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in aws" - return callback(err) - keys = _.map data.Contents, (entry)-> - return entry.Key - s3Client.deleteMultiple keys, callback + logger.log({key, bucketName}, "deleting directory"); + const s3Client = getKnoxClient(bucketName); + return s3Client.list({prefix:key}, function(err, data){ + if (err != null) { + logger.err({err, bucketName, key}, "something went wrong listing prefix in aws"); + return callback(err); + } + const keys = _.map(data.Contents, entry => entry.Key); + return s3Client.deleteMultiple(keys, callback); + }); + }, - checkIfFileExists:(bucketName, key, callback)-> - logger.log bucketName:bucketName, key:key, "checking if file exists in s3" - options = buildDefaultOptions(bucketName, "head", key) - request options, (err, res)-> - if err? - logger.err err:err, res:res, bucketName:bucketName, key:key, "something went wrong checking file in aws" - return callback(err) - if !res? - logger.err err:err, res:res, bucketName:bucketName, key:key, "no response object returned when checking if file exists" - err = new Error("no response from s3 #{bucketName} #{key}") - return callback(err) - exists = res.statusCode == 200 - logger.log bucketName:bucketName, key:key, exists:exists, "checked if file exsists in s3" - callback(err, exists) + checkIfFileExists(bucketName, key, callback){ + logger.log({bucketName, key}, "checking if file exists in s3"); + const options = buildDefaultOptions(bucketName, "head", key); + return request(options, function(err, res){ + if (err != null) { + logger.err({err, res, bucketName, key}, "something went wrong checking file in aws"); + return callback(err); + } + if ((res == null)) { + logger.err({err, res, bucketName, key}, "no response object returned when checking if file exists"); + err = new Error(`no response from s3 ${bucketName} ${key}`); + return callback(err); + } + const exists = res.statusCode === 200; + logger.log({bucketName, key, exists}, "checked if file exsists in s3"); + return callback(err, exists); + }); + }, - directorySize:(bucketName, key, callback)-> - logger.log bucketName:bucketName, key:key, "get project size in s3" - s3Client = getKnoxClient(bucketName) - s3Client.list prefix:key, (err, data)-> - if err? - logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in aws" - return callback(err) - totalSize = 0 - _.each data.Contents, (entry)-> - totalSize += entry.Size - logger.log totalSize:totalSize, "total size" - callback null, totalSize + directorySize(bucketName, key, callback){ + logger.log({bucketName, key}, "get project size in s3"); + const s3Client = getKnoxClient(bucketName); + return s3Client.list({prefix:key}, function(err, data){ + if (err != null) { + logger.err({err, bucketName, key}, "something went wrong listing prefix in aws"); + return callback(err); + } + let totalSize = 0; + _.each(data.Contents, entry => totalSize += entry.Size); + logger.log({totalSize}, "total size"); + return callback(null, totalSize); + }); + } +}; diff --git a/services/filestore/app/coffee/SafeExec.js b/services/filestore/app/coffee/SafeExec.js index 3559d35c95..b64d5376d5 100644 --- a/services/filestore/app/coffee/SafeExec.js +++ b/services/filestore/app/coffee/SafeExec.js @@ -1,48 +1,60 @@ -_ = require("underscore") -logger = require("logger-sharelatex") -child_process = require('child_process') -Settings = require "settings-sharelatex" +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const _ = require("underscore"); +const logger = require("logger-sharelatex"); +const child_process = require('child_process'); +const Settings = require("settings-sharelatex"); -# execute a command in the same way as 'exec' but with a timeout that -# kills all child processes -# -# we spawn the command with 'detached:true' to make a new process -# group, then we can kill everything in that process group. +// execute a command in the same way as 'exec' but with a timeout that +// kills all child processes +// +// we spawn the command with 'detached:true' to make a new process +// group, then we can kill everything in that process group. -module.exports = (command, options, callback = (err, stdout, stderr) ->) -> - if !Settings.enableConversions - error = new Error("Image conversions are disabled") - return callback(error) +module.exports = function(command, options, callback) { + if (callback == null) { callback = function(err, stdout, stderr) {}; } + if (!Settings.enableConversions) { + const error = new Error("Image conversions are disabled"); + return callback(error); + } - # options are {timeout: number-of-milliseconds, killSignal: signal-name} - [cmd, args...] = command + // options are {timeout: number-of-milliseconds, killSignal: signal-name} + const [cmd, ...args] = Array.from(command); - child = child_process.spawn cmd, args, {detached:true} - stdout = "" - stderr = "" + const child = child_process.spawn(cmd, args, {detached:true}); + let stdout = ""; + let stderr = ""; - cleanup = _.once (err) -> - clearTimeout killTimer if killTimer? - callback err, stdout, stderr + const cleanup = _.once(function(err) { + if (killTimer != null) { clearTimeout(killTimer); } + return callback(err, stdout, stderr); + }); - if options.timeout? - killTimer = setTimeout () -> - try - # use negative process id to kill process group - process.kill -child.pid, options.killSignal || "SIGTERM" - catch error - logger.log process: child.pid, kill_error: error, "error killing process" - , options.timeout + if (options.timeout != null) { + var killTimer = setTimeout(function() { + try { + // use negative process id to kill process group + return process.kill(-child.pid, options.killSignal || "SIGTERM"); + } catch (error) { + return logger.log({process: child.pid, kill_error: error}, "error killing process"); + } + } + , options.timeout); + } - child.on 'close', (code, signal) -> - err = if code then new Error("exit status #{code}") else signal - cleanup err + child.on('close', function(code, signal) { + const err = code ? new Error(`exit status ${code}`) : signal; + return cleanup(err); + }); - child.on 'error', (err) -> - cleanup err + child.on('error', err => cleanup(err)); - child.stdout.on 'data', (chunk) -> - stdout += chunk + child.stdout.on('data', chunk => stdout += chunk); - child.stderr.on 'data', (chunk) -> - stderr += chunk + return child.stderr.on('data', chunk => stderr += chunk); +}; From fcbb13b5bfeed7b5953c896414d5b3293275db86 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Mon, 16 Dec 2019 10:24:37 +0000 Subject: [PATCH 08/73] decaffeinate: Run post-processing cleanups on AWSSDKPersistorManager.coffee and 13 other files --- .../filestore/app/coffee/AWSSDKPersistorManager.js | 8 +++++++- services/filestore/app/coffee/BucketController.js | 5 +++++ services/filestore/app/coffee/Errors.js | 6 ++++++ services/filestore/app/coffee/FSPersistorManager.js | 7 +++++++ services/filestore/app/coffee/FileController.js | 6 ++++++ services/filestore/app/coffee/FileConverter.js | 5 +++++ services/filestore/app/coffee/FileHandler.js | 7 +++++++ .../filestore/app/coffee/HealthCheckController.js | 2 ++ services/filestore/app/coffee/ImageOptimiser.js | 5 +++++ services/filestore/app/coffee/KeyBuilder.js | 7 +++++++ services/filestore/app/coffee/LocalFileWriter.js | 5 +++++ services/filestore/app/coffee/PersistorManager.js | 2 ++ services/filestore/app/coffee/S3PersistorManager.js | 12 +++++++++++- services/filestore/app/coffee/SafeExec.js | 7 +++++++ 14 files changed, 82 insertions(+), 2 deletions(-) diff --git a/services/filestore/app/coffee/AWSSDKPersistorManager.js b/services/filestore/app/coffee/AWSSDKPersistorManager.js index c263936073..8220af4871 100644 --- a/services/filestore/app/coffee/AWSSDKPersistorManager.js +++ b/services/filestore/app/coffee/AWSSDKPersistorManager.js @@ -1,3 +1,9 @@ +/* eslint-disable + handle-callback-err, + no-return-assign, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -48,7 +54,7 @@ module.exports = { Key: key }; if ((opts.start != null) && (opts.end != null)) { - params['Range'] = `bytes=${opts.start}-${opts.end}`; + params.Range = `bytes=${opts.start}-${opts.end}`; } const request = s3.getObject(params); const stream = request.createReadStream(); diff --git a/services/filestore/app/coffee/BucketController.js b/services/filestore/app/coffee/BucketController.js index ecd4f9a54f..d2c4c06fae 100644 --- a/services/filestore/app/coffee/BucketController.js +++ b/services/filestore/app/coffee/BucketController.js @@ -1,3 +1,8 @@ +/* eslint-disable + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/filestore/app/coffee/Errors.js b/services/filestore/app/coffee/Errors.js index c4f8f7004c..4a9f65bb4b 100644 --- a/services/filestore/app/coffee/Errors.js +++ b/services/filestore/app/coffee/Errors.js @@ -1,3 +1,9 @@ +/* eslint-disable + no-proto, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. let Errors; var NotFoundError = function(message) { const error = new Error(message); diff --git a/services/filestore/app/coffee/FSPersistorManager.js b/services/filestore/app/coffee/FSPersistorManager.js index c4792e6215..c90d4b6dfe 100644 --- a/services/filestore/app/coffee/FSPersistorManager.js +++ b/services/filestore/app/coffee/FSPersistorManager.js @@ -1,3 +1,10 @@ +/* eslint-disable + handle-callback-err, + no-unreachable, + node/no-deprecated-api, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/filestore/app/coffee/FileController.js b/services/filestore/app/coffee/FileController.js index f40d4a0317..51251ba010 100644 --- a/services/filestore/app/coffee/FileController.js +++ b/services/filestore/app/coffee/FileController.js @@ -1,3 +1,9 @@ +/* eslint-disable + camelcase, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/filestore/app/coffee/FileConverter.js b/services/filestore/app/coffee/FileConverter.js index b35265c22b..e241cd8275 100644 --- a/services/filestore/app/coffee/FileConverter.js +++ b/services/filestore/app/coffee/FileConverter.js @@ -1,3 +1,8 @@ +/* eslint-disable + camelcase, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/filestore/app/coffee/FileHandler.js b/services/filestore/app/coffee/FileHandler.js index 548e206a12..c22a276b65 100644 --- a/services/filestore/app/coffee/FileHandler.js +++ b/services/filestore/app/coffee/FileHandler.js @@ -1,3 +1,10 @@ +/* eslint-disable + camelcase, + no-self-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/filestore/app/coffee/HealthCheckController.js b/services/filestore/app/coffee/HealthCheckController.js index 4f49f4a19a..e360891609 100644 --- a/services/filestore/app/coffee/HealthCheckController.js +++ b/services/filestore/app/coffee/HealthCheckController.js @@ -1,3 +1,5 @@ +// TODO: This file was created by bulk-decaffeinate. +// Sanity-check the conversion and remove this comment. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/filestore/app/coffee/ImageOptimiser.js b/services/filestore/app/coffee/ImageOptimiser.js index 570730618d..0126786167 100644 --- a/services/filestore/app/coffee/ImageOptimiser.js +++ b/services/filestore/app/coffee/ImageOptimiser.js @@ -1,3 +1,8 @@ +/* eslint-disable + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/filestore/app/coffee/KeyBuilder.js b/services/filestore/app/coffee/KeyBuilder.js index f5c8b7d5da..5d3a36e6b1 100644 --- a/services/filestore/app/coffee/KeyBuilder.js +++ b/services/filestore/app/coffee/KeyBuilder.js @@ -1,3 +1,10 @@ +/* eslint-disable + camelcase, + no-return-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/filestore/app/coffee/LocalFileWriter.js b/services/filestore/app/coffee/LocalFileWriter.js index a6457de526..b0bfedcc04 100644 --- a/services/filestore/app/coffee/LocalFileWriter.js +++ b/services/filestore/app/coffee/LocalFileWriter.js @@ -1,3 +1,8 @@ +/* eslint-disable + handle-callback-err, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/filestore/app/coffee/PersistorManager.js b/services/filestore/app/coffee/PersistorManager.js index 993a874d6a..95a4122591 100644 --- a/services/filestore/app/coffee/PersistorManager.js +++ b/services/filestore/app/coffee/PersistorManager.js @@ -1,3 +1,5 @@ +// TODO: This file was created by bulk-decaffeinate. +// Sanity-check the conversion and remove this comment. /* * decaffeinate suggestions: * DS103: Rewrite code to no longer use __guard__ diff --git a/services/filestore/app/coffee/S3PersistorManager.js b/services/filestore/app/coffee/S3PersistorManager.js index a8edbc6844..0face2de53 100644 --- a/services/filestore/app/coffee/S3PersistorManager.js +++ b/services/filestore/app/coffee/S3PersistorManager.js @@ -1,3 +1,13 @@ +/* eslint-disable + handle-callback-err, + new-cap, + no-return-assign, + no-unused-vars, + node/no-deprecated-api, + standard/no-callback-literal, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from @@ -146,7 +156,7 @@ module.exports = { Key: key }; if ((opts.start != null) && (opts.end != null)) { - s3Params['Range'] = `bytes=${opts.start}-${opts.end}`; + s3Params.Range = `bytes=${opts.start}-${opts.end}`; } const s3Request = s3.getObject(s3Params); diff --git a/services/filestore/app/coffee/SafeExec.js b/services/filestore/app/coffee/SafeExec.js index b64d5376d5..4f13785460 100644 --- a/services/filestore/app/coffee/SafeExec.js +++ b/services/filestore/app/coffee/SafeExec.js @@ -1,3 +1,10 @@ +/* eslint-disable + camelcase, + handle-callback-err, + no-return-assign, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from From 3718c90c7a5b8fa676fd00b047c5edbed930d911 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 16 Dec 2019 10:24:40 +0000 Subject: [PATCH 09/73] Rename app/coffee dir to app/js --- services/filestore/.gitignore | 17 ----------------- .../{coffee => js}/AWSSDKPersistorManager.js | 0 .../app/{coffee => js}/BucketController.js | 0 services/filestore/app/{coffee => js}/Errors.js | 0 .../app/{coffee => js}/FSPersistorManager.js | 0 .../app/{coffee => js}/FileController.js | 0 .../app/{coffee => js}/FileConverter.js | 0 .../filestore/app/{coffee => js}/FileHandler.js | 0 .../app/{coffee => js}/HealthCheckController.js | 0 .../app/{coffee => js}/ImageOptimiser.js | 0 .../filestore/app/{coffee => js}/KeyBuilder.js | 0 .../app/{coffee => js}/LocalFileWriter.js | 0 .../app/{coffee => js}/PersistorManager.js | 0 .../app/{coffee => js}/S3PersistorManager.js | 0 .../filestore/app/{coffee => js}/SafeExec.js | 0 15 files changed, 17 deletions(-) rename services/filestore/app/{coffee => js}/AWSSDKPersistorManager.js (100%) rename services/filestore/app/{coffee => js}/BucketController.js (100%) rename services/filestore/app/{coffee => js}/Errors.js (100%) rename services/filestore/app/{coffee => js}/FSPersistorManager.js (100%) rename services/filestore/app/{coffee => js}/FileController.js (100%) rename services/filestore/app/{coffee => js}/FileConverter.js (100%) rename services/filestore/app/{coffee => js}/FileHandler.js (100%) rename services/filestore/app/{coffee => js}/HealthCheckController.js (100%) rename services/filestore/app/{coffee => js}/ImageOptimiser.js (100%) rename services/filestore/app/{coffee => js}/KeyBuilder.js (100%) rename services/filestore/app/{coffee => js}/LocalFileWriter.js (100%) rename services/filestore/app/{coffee => js}/PersistorManager.js (100%) rename services/filestore/app/{coffee => js}/S3PersistorManager.js (100%) rename services/filestore/app/{coffee => js}/SafeExec.js (100%) diff --git a/services/filestore/.gitignore b/services/filestore/.gitignore index 5d1086933c..3c0b90c20d 100644 --- a/services/filestore/.gitignore +++ b/services/filestore/.gitignore @@ -36,28 +36,11 @@ Icon? Thumbs.db /node_modules/* -test/IntergrationTests/js/* data/*/* -app.js **/*.map -cluster.js -app/js/* -test/IntergrationTests/js/* -test/UnitTests/js/* cookies.txt uploads/* -public/js/editor.js -public/js/home.js -public/js/forms.js -public/js/gui.js -public/js/admin.js -public/stylesheets/mainStyle.css -public/minjs/ -test/unit/js/ -test/acceptence/js -cluster.js -test/acceptance/js/ user_files/* template_files/* diff --git a/services/filestore/app/coffee/AWSSDKPersistorManager.js b/services/filestore/app/js/AWSSDKPersistorManager.js similarity index 100% rename from services/filestore/app/coffee/AWSSDKPersistorManager.js rename to services/filestore/app/js/AWSSDKPersistorManager.js diff --git a/services/filestore/app/coffee/BucketController.js b/services/filestore/app/js/BucketController.js similarity index 100% rename from services/filestore/app/coffee/BucketController.js rename to services/filestore/app/js/BucketController.js diff --git a/services/filestore/app/coffee/Errors.js b/services/filestore/app/js/Errors.js similarity index 100% rename from services/filestore/app/coffee/Errors.js rename to services/filestore/app/js/Errors.js diff --git a/services/filestore/app/coffee/FSPersistorManager.js b/services/filestore/app/js/FSPersistorManager.js similarity index 100% rename from services/filestore/app/coffee/FSPersistorManager.js rename to services/filestore/app/js/FSPersistorManager.js diff --git a/services/filestore/app/coffee/FileController.js b/services/filestore/app/js/FileController.js similarity index 100% rename from services/filestore/app/coffee/FileController.js rename to services/filestore/app/js/FileController.js diff --git a/services/filestore/app/coffee/FileConverter.js b/services/filestore/app/js/FileConverter.js similarity index 100% rename from services/filestore/app/coffee/FileConverter.js rename to services/filestore/app/js/FileConverter.js diff --git a/services/filestore/app/coffee/FileHandler.js b/services/filestore/app/js/FileHandler.js similarity index 100% rename from services/filestore/app/coffee/FileHandler.js rename to services/filestore/app/js/FileHandler.js diff --git a/services/filestore/app/coffee/HealthCheckController.js b/services/filestore/app/js/HealthCheckController.js similarity index 100% rename from services/filestore/app/coffee/HealthCheckController.js rename to services/filestore/app/js/HealthCheckController.js diff --git a/services/filestore/app/coffee/ImageOptimiser.js b/services/filestore/app/js/ImageOptimiser.js similarity index 100% rename from services/filestore/app/coffee/ImageOptimiser.js rename to services/filestore/app/js/ImageOptimiser.js diff --git a/services/filestore/app/coffee/KeyBuilder.js b/services/filestore/app/js/KeyBuilder.js similarity index 100% rename from services/filestore/app/coffee/KeyBuilder.js rename to services/filestore/app/js/KeyBuilder.js diff --git a/services/filestore/app/coffee/LocalFileWriter.js b/services/filestore/app/js/LocalFileWriter.js similarity index 100% rename from services/filestore/app/coffee/LocalFileWriter.js rename to services/filestore/app/js/LocalFileWriter.js diff --git a/services/filestore/app/coffee/PersistorManager.js b/services/filestore/app/js/PersistorManager.js similarity index 100% rename from services/filestore/app/coffee/PersistorManager.js rename to services/filestore/app/js/PersistorManager.js diff --git a/services/filestore/app/coffee/S3PersistorManager.js b/services/filestore/app/js/S3PersistorManager.js similarity index 100% rename from services/filestore/app/coffee/S3PersistorManager.js rename to services/filestore/app/js/S3PersistorManager.js diff --git a/services/filestore/app/coffee/SafeExec.js b/services/filestore/app/js/SafeExec.js similarity index 100% rename from services/filestore/app/coffee/SafeExec.js rename to services/filestore/app/js/SafeExec.js From 601861ee58951611c1885a2af1946f93ab22948f Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 16 Dec 2019 10:34:51 +0000 Subject: [PATCH 10/73] Remove js from dockerignore --- services/filestore/.dockerignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/filestore/.dockerignore b/services/filestore/.dockerignore index 386f26df30..ba1c3442de 100644 --- a/services/filestore/.dockerignore +++ b/services/filestore/.dockerignore @@ -5,5 +5,3 @@ gitrev .npm .nvmrc nodemon.json -app.js -**/js/* From 6225f2f236db327a56440f97de5e70f511217190 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 16 Dec 2019 10:42:31 +0000 Subject: [PATCH 11/73] Prettier: convert app/js decaffeinated files to Prettier format --- .../app/js/AWSSDKPersistorManager.js | 294 +++++---- services/filestore/app/js/BucketController.js | 70 ++- services/filestore/app/js/Errors.js | 17 +- .../filestore/app/js/FSPersistorManager.js | 239 +++++--- services/filestore/app/js/FileController.js | 284 +++++---- services/filestore/app/js/FileConverter.js | 182 ++++-- services/filestore/app/js/FileHandler.js | 359 ++++++----- .../filestore/app/js/HealthCheckController.js | 132 ++-- services/filestore/app/js/ImageOptimiser.js | 64 +- services/filestore/app/js/KeyBuilder.js | 106 ++-- services/filestore/app/js/LocalFileWriter.js | 142 +++-- services/filestore/app/js/PersistorManager.js | 54 +- .../filestore/app/js/S3PersistorManager.js | 580 ++++++++++-------- services/filestore/app/js/SafeExec.js | 80 +-- 14 files changed, 1464 insertions(+), 1139 deletions(-) diff --git a/services/filestore/app/js/AWSSDKPersistorManager.js b/services/filestore/app/js/AWSSDKPersistorManager.js index 8220af4871..4dbc836280 100644 --- a/services/filestore/app/js/AWSSDKPersistorManager.js +++ b/services/filestore/app/js/AWSSDKPersistorManager.js @@ -15,137 +15,183 @@ // latest aws-sdk and delete this module so that PersistorManager would load the // same backend for both the 's3' and 'aws-sdk' options. -const logger = require("logger-sharelatex"); -const aws = require("aws-sdk"); -const _ = require("underscore"); -const fs = require("fs"); -const Errors = require("./Errors"); +const logger = require('logger-sharelatex') +const aws = require('aws-sdk') +const _ = require('underscore') +const fs = require('fs') +const Errors = require('./Errors') -const s3 = new aws.S3(); +const s3 = new aws.S3() module.exports = { - sendFile(bucketName, key, fsPath, callback){ - logger.log({bucketName, key}, "send file data to s3"); - const stream = fs.createReadStream(fsPath); - return s3.upload({Bucket: bucketName, Key: key, Body: stream}, function(err, data) { - if (err != null) { - logger.err({err, Bucket: bucketName, Key: key}, "error sending file data to s3"); - } - return callback(err); - }); - }, + sendFile(bucketName, key, fsPath, callback) { + logger.log({ bucketName, key }, 'send file data to s3') + const stream = fs.createReadStream(fsPath) + return s3.upload({ Bucket: bucketName, Key: key, Body: stream }, function( + err, + data + ) { + if (err != null) { + logger.err( + { err, Bucket: bucketName, Key: key }, + 'error sending file data to s3' + ) + } + return callback(err) + }) + }, - sendStream(bucketName, key, stream, callback){ - logger.log({bucketName, key}, "send file stream to s3"); - return s3.upload({Bucket: bucketName, Key: key, Body: stream}, function(err, data) { - if (err != null) { - logger.err({err, Bucket: bucketName, Key: key}, "error sending file stream to s3"); - } - return callback(err); - }); - }, + sendStream(bucketName, key, stream, callback) { + logger.log({ bucketName, key }, 'send file stream to s3') + return s3.upload({ Bucket: bucketName, Key: key, Body: stream }, function( + err, + data + ) { + if (err != null) { + logger.err( + { err, Bucket: bucketName, Key: key }, + 'error sending file stream to s3' + ) + } + return callback(err) + }) + }, - getFileStream(bucketName, key, opts, callback){ - if (callback == null) { callback = function(err, res){}; } - logger.log({bucketName, key}, "get file stream from s3"); - callback = _.once(callback); - const params = { - Bucket:bucketName, - Key: key - }; - if ((opts.start != null) && (opts.end != null)) { - params.Range = `bytes=${opts.start}-${opts.end}`; - } - const request = s3.getObject(params); - const stream = request.createReadStream(); - stream.on('readable', () => callback(null, stream)); - return stream.on('error', function(err) { - logger.err({err, bucketName, key}, "error getting file stream from s3"); - if (err.code === 'NoSuchKey') { - return callback(new Errors.NotFoundError(`File not found in S3: ${bucketName}:${key}`)); - } - return callback(err); - }); - }, + getFileStream(bucketName, key, opts, callback) { + if (callback == null) { + callback = function(err, res) {} + } + logger.log({ bucketName, key }, 'get file stream from s3') + callback = _.once(callback) + const params = { + Bucket: bucketName, + Key: key + } + if (opts.start != null && opts.end != null) { + params.Range = `bytes=${opts.start}-${opts.end}` + } + const request = s3.getObject(params) + const stream = request.createReadStream() + stream.on('readable', () => callback(null, stream)) + return stream.on('error', function(err) { + logger.err({ err, bucketName, key }, 'error getting file stream from s3') + if (err.code === 'NoSuchKey') { + return callback( + new Errors.NotFoundError(`File not found in S3: ${bucketName}:${key}`) + ) + } + return callback(err) + }) + }, - copyFile(bucketName, sourceKey, destKey, callback){ - logger.log({bucketName, sourceKey, destKey}, "copying file in s3"); - const source = bucketName + '/' + sourceKey; - return s3.copyObject({Bucket: bucketName, Key: destKey, CopySource: source}, function(err) { - if (err != null) { - logger.err({err, bucketName, sourceKey, destKey}, "something went wrong copying file in s3"); - } - return callback(err); - }); - }, + copyFile(bucketName, sourceKey, destKey, callback) { + logger.log({ bucketName, sourceKey, destKey }, 'copying file in s3') + const source = bucketName + '/' + sourceKey + return s3.copyObject( + { Bucket: bucketName, Key: destKey, CopySource: source }, + function(err) { + if (err != null) { + logger.err( + { err, bucketName, sourceKey, destKey }, + 'something went wrong copying file in s3' + ) + } + return callback(err) + } + ) + }, - deleteFile(bucketName, key, callback){ - logger.log({bucketName, key}, "delete file in s3"); - return s3.deleteObject({Bucket: bucketName, Key: key}, function(err) { - if (err != null) { - logger.err({err, bucketName, key}, "something went wrong deleting file in s3"); - } - return callback(err); - }); - }, + deleteFile(bucketName, key, callback) { + logger.log({ bucketName, key }, 'delete file in s3') + return s3.deleteObject({ Bucket: bucketName, Key: key }, function(err) { + if (err != null) { + logger.err( + { err, bucketName, key }, + 'something went wrong deleting file in s3' + ) + } + return callback(err) + }) + }, - deleteDirectory(bucketName, key, callback){ - logger.log({bucketName, key}, "delete directory in s3"); - return s3.listObjects({Bucket: bucketName, Prefix: key}, function(err, data) { - if (err != null) { - logger.err({err, bucketName, key}, "something went wrong listing prefix in s3"); - return callback(err); - } - if (data.Contents.length === 0) { - logger.log({bucketName, key}, "the directory is empty"); - return callback(); - } - const keys = _.map(data.Contents, entry => ({ - Key: entry.Key - })); - return s3.deleteObjects({ - Bucket: bucketName, - Delete: { - Objects: keys, - Quiet: true - } - } - , function(err) { - if (err != null) { - logger.err({err, bucketName, key:keys}, "something went wrong deleting directory in s3"); - } - return callback(err); - }); - }); - }, + deleteDirectory(bucketName, key, callback) { + logger.log({ bucketName, key }, 'delete directory in s3') + return s3.listObjects({ Bucket: bucketName, Prefix: key }, function( + err, + data + ) { + if (err != null) { + logger.err( + { err, bucketName, key }, + 'something went wrong listing prefix in s3' + ) + return callback(err) + } + if (data.Contents.length === 0) { + logger.log({ bucketName, key }, 'the directory is empty') + return callback() + } + const keys = _.map(data.Contents, entry => ({ + Key: entry.Key + })) + return s3.deleteObjects( + { + Bucket: bucketName, + Delete: { + Objects: keys, + Quiet: true + } + }, + function(err) { + if (err != null) { + logger.err( + { err, bucketName, key: keys }, + 'something went wrong deleting directory in s3' + ) + } + return callback(err) + } + ) + }) + }, - checkIfFileExists(bucketName, key, callback){ - logger.log({bucketName, key}, "check file existence in s3"); - return s3.headObject({Bucket: bucketName, Key: key}, function(err, data) { - if (err != null) { - if (err.code === 'NotFound') { return (callback(null, false)); } - logger.err({err, bucketName, key}, "something went wrong checking head in s3"); - return callback(err); - } - return callback(null, (data.ETag != null)); - }); - }, - - directorySize(bucketName, key, callback){ - logger.log({bucketName, key}, "get project size in s3"); - return s3.listObjects({Bucket: bucketName, Prefix: key}, function(err, data) { - if (err != null) { - logger.err({err, bucketName, key}, "something went wrong listing prefix in s3"); - return callback(err); - } - if (data.Contents.length === 0) { - logger.log({bucketName, key}, "the directory is empty"); - return callback(); - } - let totalSize = 0; - _.each(data.Contents, entry => totalSize += entry.Size); - return callback(null, totalSize); - }); - } -}; + checkIfFileExists(bucketName, key, callback) { + logger.log({ bucketName, key }, 'check file existence in s3') + return s3.headObject({ Bucket: bucketName, Key: key }, function(err, data) { + if (err != null) { + if (err.code === 'NotFound') { + return callback(null, false) + } + logger.err( + { err, bucketName, key }, + 'something went wrong checking head in s3' + ) + return callback(err) + } + return callback(null, data.ETag != null) + }) + }, + directorySize(bucketName, key, callback) { + logger.log({ bucketName, key }, 'get project size in s3') + return s3.listObjects({ Bucket: bucketName, Prefix: key }, function( + err, + data + ) { + if (err != null) { + logger.err( + { err, bucketName, key }, + 'something went wrong listing prefix in s3' + ) + return callback(err) + } + if (data.Contents.length === 0) { + logger.log({ bucketName, key }, 'the directory is empty') + return callback() + } + let totalSize = 0 + _.each(data.Contents, entry => (totalSize += entry.Size)) + return callback(null, totalSize) + }) + } +} diff --git a/services/filestore/app/js/BucketController.js b/services/filestore/app/js/BucketController.js index d2c4c06fae..46f69679aa 100644 --- a/services/filestore/app/js/BucketController.js +++ b/services/filestore/app/js/BucketController.js @@ -9,38 +9,40 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let BucketController; -const settings = require("settings-sharelatex"); -const logger = require("logger-sharelatex"); -const FileHandler = require("./FileHandler"); -const metrics = require("metrics-sharelatex"); -const Errors = require('./Errors'); +let BucketController +const settings = require('settings-sharelatex') +const logger = require('logger-sharelatex') +const FileHandler = require('./FileHandler') +const metrics = require('metrics-sharelatex') +const Errors = require('./Errors') -module.exports = (BucketController = { - - getFile(req, res){ - const {bucket} = req.params; - const key = req.params[0]; - const credentials = settings.filestore.s3BucketCreds != null ? settings.filestore.s3BucketCreds[bucket] : undefined; - const options = { - key, - bucket, - credentials - }; - metrics.inc(`${bucket}.getFile`); - logger.log({key, bucket}, "receiving request to get file from bucket"); - return FileHandler.getFile(bucket, key, options, function(err, fileStream){ - if (err != null) { - logger.err({err, key, bucket}, "problem getting file from bucket"); - if (err instanceof Errors.NotFoundError) { - return res.send(404); - } else { - return res.send(500); - } - } else { - logger.log({key, bucket}, "sending bucket file to response"); - return fileStream.pipe(res); - } - }); - } -}); +module.exports = BucketController = { + getFile(req, res) { + const { bucket } = req.params + const key = req.params[0] + const credentials = + settings.filestore.s3BucketCreds != null + ? settings.filestore.s3BucketCreds[bucket] + : undefined + const options = { + key, + bucket, + credentials + } + metrics.inc(`${bucket}.getFile`) + logger.log({ key, bucket }, 'receiving request to get file from bucket') + return FileHandler.getFile(bucket, key, options, function(err, fileStream) { + if (err != null) { + logger.err({ err, key, bucket }, 'problem getting file from bucket') + if (err instanceof Errors.NotFoundError) { + return res.send(404) + } else { + return res.send(500) + } + } else { + logger.log({ key, bucket }, 'sending bucket file to response') + return fileStream.pipe(res) + } + }) + } +} diff --git a/services/filestore/app/js/Errors.js b/services/filestore/app/js/Errors.js index 4a9f65bb4b..f3bc8e37eb 100644 --- a/services/filestore/app/js/Errors.js +++ b/services/filestore/app/js/Errors.js @@ -4,14 +4,13 @@ */ // TODO: This file was created by bulk-decaffeinate. // Fix any style issues and re-enable lint. -let Errors; +let Errors var NotFoundError = function(message) { - const error = new Error(message); - error.name = "NotFoundError"; - error.__proto__ = NotFoundError.prototype; - return error; -}; -NotFoundError.prototype.__proto__ = Error.prototype; + const error = new Error(message) + error.name = 'NotFoundError' + error.__proto__ = NotFoundError.prototype + return error +} +NotFoundError.prototype.__proto__ = Error.prototype -module.exports = (Errors = - {NotFoundError}); +module.exports = Errors = { NotFoundError } diff --git a/services/filestore/app/js/FSPersistorManager.js b/services/filestore/app/js/FSPersistorManager.js index c90d4b6dfe..2e0de65632 100644 --- a/services/filestore/app/js/FSPersistorManager.js +++ b/services/filestore/app/js/FSPersistorManager.js @@ -11,161 +11,196 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const logger = require("logger-sharelatex"); -const fs = require("fs"); -const path = require("path"); -const LocalFileWriter = require("./LocalFileWriter"); -const Errors = require('./Errors'); -const rimraf = require("rimraf"); -const _ = require("underscore"); - -const filterName = key => key.replace(/\//g, "_"); +const logger = require('logger-sharelatex') +const fs = require('fs') +const path = require('path') +const LocalFileWriter = require('./LocalFileWriter') +const Errors = require('./Errors') +const rimraf = require('rimraf') +const _ = require('underscore') +const filterName = key => key.replace(/\//g, '_') module.exports = { - sendFile( location, target, source, callback) { - if (callback == null) { callback = function(err){}; } - const filteredTarget = filterName(target); - logger.log({location, target:filteredTarget, source}, "sending file"); + sendFile(location, target, source, callback) { + if (callback == null) { + callback = function(err) {} + } + const filteredTarget = filterName(target) + logger.log({ location, target: filteredTarget, source }, 'sending file') const done = _.once(function(err) { if (err != null) { - logger.err({err, location, target:filteredTarget, source}, "Error on put of file"); + logger.err( + { err, location, target: filteredTarget, source }, + 'Error on put of file' + ) } - return callback(err); - }); + return callback(err) + }) // actually copy the file (instead of moving it) to maintain consistent behaviour // between the different implementations - const sourceStream = fs.createReadStream(source); - sourceStream.on('error', done); - const targetStream = fs.createWriteStream(`${location}/${filteredTarget}`); - targetStream.on('error', done); - targetStream.on('finish', () => done()); - return sourceStream.pipe(targetStream); + const sourceStream = fs.createReadStream(source) + sourceStream.on('error', done) + const targetStream = fs.createWriteStream(`${location}/${filteredTarget}`) + targetStream.on('error', done) + targetStream.on('finish', () => done()) + return sourceStream.pipe(targetStream) }, - sendStream( location, target, sourceStream, callback) { - if (callback == null) { callback = function(err){}; } - logger.log({location, target}, "sending file stream"); - sourceStream.on("error", err => logger.err({location, target, err:err("error on stream to send")})); - return LocalFileWriter.writeStream(sourceStream, null, (err, fsPath)=> { + sendStream(location, target, sourceStream, callback) { + if (callback == null) { + callback = function(err) {} + } + logger.log({ location, target }, 'sending file stream') + sourceStream.on('error', err => + logger.err({ location, target, err: err('error on stream to send') }) + ) + return LocalFileWriter.writeStream(sourceStream, null, (err, fsPath) => { if (err != null) { - logger.err({location, target, fsPath, err}, "something went wrong writing stream to disk"); - return callback(err); + logger.err( + { location, target, fsPath, err }, + 'something went wrong writing stream to disk' + ) + return callback(err) } - return this.sendFile(location, target, fsPath, err => // delete the temporary file created above and return the original error - LocalFileWriter.deleteFile(fsPath, () => callback(err))); - }); + return this.sendFile(location, target, fsPath, ( + err // delete the temporary file created above and return the original error + ) => LocalFileWriter.deleteFile(fsPath, () => callback(err))) + }) }, // opts may be {start: Number, end: Number} getFileStream(location, name, opts, callback) { - if (callback == null) { callback = function(err, res){}; } - const filteredName = filterName(name); - logger.log({location, filteredName}, "getting file"); + if (callback == null) { + callback = function(err, res) {} + } + const filteredName = filterName(name) + logger.log({ location, filteredName }, 'getting file') return fs.open(`${location}/${filteredName}`, 'r', function(err, fd) { if (err != null) { - logger.err({err, location, filteredName:name}, "Error reading from file"); + logger.err( + { err, location, filteredName: name }, + 'Error reading from file' + ) } if (err.code === 'ENOENT') { - return callback(new Errors.NotFoundError(err.message), null); + return callback(new Errors.NotFoundError(err.message), null) } else { - return callback(err, null); + return callback(err, null) } - opts.fd = fd; - const sourceStream = fs.createReadStream(null, opts); - return callback(null, sourceStream); - }); + opts.fd = fd + const sourceStream = fs.createReadStream(null, opts) + return callback(null, sourceStream) + }) }, getFileSize(location, filename, callback) { - const fullPath = path.join(location, filterName(filename)); + const fullPath = path.join(location, filterName(filename)) return fs.stat(fullPath, function(err, stats) { if (err != null) { if (err.code === 'ENOENT') { - logger.log({location, filename}, "file not found"); - callback(new Errors.NotFoundError(err.message)); + logger.log({ location, filename }, 'file not found') + callback(new Errors.NotFoundError(err.message)) } else { - logger.err({err, location, filename}, "failed to stat file"); - callback(err); + logger.err({ err, location, filename }, 'failed to stat file') + callback(err) } - return; + return } - return callback(null, stats.size); - }); + return callback(null, stats.size) + }) }, - copyFile(location, fromName, toName, callback){ - if (callback == null) { callback = function(err){}; } - const filteredFromName=filterName(fromName); - const filteredToName=filterName(toName); - logger.log({location, fromName:filteredFromName, toName:filteredToName}, "copying file"); - const sourceStream = fs.createReadStream(`${location}/${filteredFromName}`); + copyFile(location, fromName, toName, callback) { + if (callback == null) { + callback = function(err) {} + } + const filteredFromName = filterName(fromName) + const filteredToName = filterName(toName) + logger.log( + { location, fromName: filteredFromName, toName: filteredToName }, + 'copying file' + ) + const sourceStream = fs.createReadStream(`${location}/${filteredFromName}`) sourceStream.on('error', function(err) { - logger.err({err, location, key:filteredFromName}, "Error reading from file"); - return callback(err); - }); - const targetStream = fs.createWriteStream(`${location}/${filteredToName}`); + logger.err( + { err, location, key: filteredFromName }, + 'Error reading from file' + ) + return callback(err) + }) + const targetStream = fs.createWriteStream(`${location}/${filteredToName}`) targetStream.on('error', function(err) { - logger.err({err, location, key:filteredToName}, "Error writing to file"); - return callback(err); - }); - targetStream.on('finish', () => callback(null)); - return sourceStream.pipe(targetStream); + logger.err( + { err, location, key: filteredToName }, + 'Error writing to file' + ) + return callback(err) + }) + targetStream.on('finish', () => callback(null)) + return sourceStream.pipe(targetStream) }, - deleteFile(location, name, callback){ - const filteredName = filterName(name); - logger.log({location, filteredName}, "delete file"); + deleteFile(location, name, callback) { + const filteredName = filterName(name) + logger.log({ location, filteredName }, 'delete file') return fs.unlink(`${location}/${filteredName}`, function(err) { if (err != null) { - logger.err({err, location, filteredName}, "Error on delete."); - return callback(err); + logger.err({ err, location, filteredName }, 'Error on delete.') + return callback(err) } else { - return callback(); + return callback() } - }); + }) }, - deleteDirectory(location, name, callback){ - if (callback == null) { callback = function(err){}; } - const filteredName = filterName(name.replace(/\/$/,'')); + deleteDirectory(location, name, callback) { + if (callback == null) { + callback = function(err) {} + } + const filteredName = filterName(name.replace(/\/$/, '')) return rimraf(`${location}/${filteredName}`, function(err) { if (err != null) { - logger.err({err, location, filteredName}, "Error on rimraf rmdir."); - return callback(err); + logger.err({ err, location, filteredName }, 'Error on rimraf rmdir.') + return callback(err) } else { - return callback(); + return callback() } - }); + }) }, - checkIfFileExists(location, name, callback){ - if (callback == null) { callback = function(err,exists){}; } - const filteredName = filterName(name); - logger.log({location, filteredName}, "checking if file exists"); + checkIfFileExists(location, name, callback) { + if (callback == null) { + callback = function(err, exists) {} + } + const filteredName = filterName(name) + logger.log({ location, filteredName }, 'checking if file exists') return fs.exists(`${location}/${filteredName}`, function(exists) { - logger.log({location, filteredName, exists}, "checked if file exists"); - return callback(null, exists); - }); + logger.log({ location, filteredName, exists }, 'checked if file exists') + return callback(null, exists) + }) }, - directorySize(location, name, callback){ - const filteredName = filterName(name.replace(/\/$/,'')); - logger.log({location, filteredName}, "get project size in file system"); + directorySize(location, name, callback) { + const filteredName = filterName(name.replace(/\/$/, '')) + logger.log({ location, filteredName }, 'get project size in file system') return fs.readdir(`${location}/${filteredName}`, function(err, files) { if (err != null) { - logger.err({err, location, filteredName}, "something went wrong listing prefix in aws"); - return callback(err); + logger.err( + { err, location, filteredName }, + 'something went wrong listing prefix in aws' + ) + return callback(err) } - let totalSize = 0; - _.each(files, function(entry){ - const fd = fs.openSync(`${location}/${filteredName}/${entry}`, 'r'); - const fileStats = fs.fstatSync(fd); - totalSize += fileStats.size; - return fs.closeSync(fd); - }); - logger.log({totalSize}, "total size", {files}); - return callback(null, totalSize); - }); + let totalSize = 0 + _.each(files, function(entry) { + const fd = fs.openSync(`${location}/${filteredName}/${entry}`, 'r') + const fileStats = fs.fstatSync(fd) + totalSize += fileStats.size + return fs.closeSync(fd) + }) + logger.log({ totalSize }, 'total size', { files }) + return callback(null, totalSize) + }) } -}; +} diff --git a/services/filestore/app/js/FileController.js b/services/filestore/app/js/FileController.js index 51251ba010..ea41f3d813 100644 --- a/services/filestore/app/js/FileController.js +++ b/services/filestore/app/js/FileController.js @@ -10,142 +10,164 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let FileController; -const PersistorManager = require("./PersistorManager"); -const settings = require("settings-sharelatex"); -const logger = require("logger-sharelatex"); -const FileHandler = require("./FileHandler"); -const metrics = require("metrics-sharelatex"); -const parseRange = require('range-parser'); -const Errors = require('./Errors'); +let FileController +const PersistorManager = require('./PersistorManager') +const settings = require('settings-sharelatex') +const logger = require('logger-sharelatex') +const FileHandler = require('./FileHandler') +const metrics = require('metrics-sharelatex') +const parseRange = require('range-parser') +const Errors = require('./Errors') -const oneDayInSeconds = 60 * 60 * 24; -const maxSizeInBytes = 1024 * 1024 * 1024; // 1GB +const oneDayInSeconds = 60 * 60 * 24 +const maxSizeInBytes = 1024 * 1024 * 1024 // 1GB -module.exports = (FileController = { +module.exports = FileController = { + getFile(req, res) { + const { key, bucket } = req + const { format, style } = req.query + const options = { + key, + bucket, + format, + style + } + metrics.inc('getFile') + logger.log({ key, bucket, format, style }, 'receiving request to get file') + if (req.headers.range != null) { + const range = FileController._get_range(req.headers.range) + options.start = range.start + options.end = range.end + logger.log( + { start: range.start, end: range.end }, + 'getting range of bytes from file' + ) + } + return FileHandler.getFile(bucket, key, options, function(err, fileStream) { + if (err != null) { + if (err instanceof Errors.NotFoundError) { + return res.send(404) + } else { + logger.err( + { err, key, bucket, format, style }, + 'problem getting file' + ) + return res.send(500) + } + } else if (req.query.cacheWarm) { + logger.log( + { key, bucket, format, style }, + 'request is only for cache warm so not sending stream' + ) + return res.send(200) + } else { + logger.log({ key, bucket, format, style }, 'sending file to response') + return fileStream.pipe(res) + } + }) + }, - getFile(req, res){ - const {key, bucket} = req; - const {format, style} = req.query; - const options = { - key, - bucket, - format, - style, - }; - metrics.inc("getFile"); - logger.log({key, bucket, format, style}, "receiving request to get file"); - if (req.headers.range != null) { - const range = FileController._get_range(req.headers.range); - options.start = range.start; - options.end = range.end; - logger.log({start: range.start, end: range.end}, "getting range of bytes from file"); - } - return FileHandler.getFile(bucket, key, options, function(err, fileStream){ - if (err != null) { - if (err instanceof Errors.NotFoundError) { - return res.send(404); - } else { - logger.err({err, key, bucket, format, style}, "problem getting file"); - return res.send(500); - } - } else if (req.query.cacheWarm) { - logger.log({key, bucket, format, style}, "request is only for cache warm so not sending stream"); - return res.send(200); - } else { - logger.log({key, bucket, format, style}, "sending file to response"); - return fileStream.pipe(res); - } - }); - }, + getFileHead(req, res) { + const { key, bucket } = req + metrics.inc('getFileSize') + logger.log({ key, bucket }, 'receiving request to get file metadata') + return FileHandler.getFileSize(bucket, key, function(err, fileSize) { + if (err != null) { + if (err instanceof Errors.NotFoundError) { + res.status(404).end() + } else { + res.status(500).end() + } + return + } + res.set('Content-Length', fileSize) + return res.status(200).end() + }) + }, - getFileHead(req, res) { - const {key, bucket} = req; - metrics.inc("getFileSize"); - logger.log({ key, bucket }, "receiving request to get file metadata"); - return FileHandler.getFileSize(bucket, key, function(err, fileSize) { - if (err != null) { - if (err instanceof Errors.NotFoundError) { - res.status(404).end(); - } else { - res.status(500).end(); - } - return; - } - res.set("Content-Length", fileSize); - return res.status(200).end(); - }); - }, + insertFile(req, res) { + metrics.inc('insertFile') + const { key, bucket } = req + logger.log({ key, bucket }, 'receiving request to insert file') + return FileHandler.insertFile(bucket, key, req, function(err) { + if (err != null) { + logger.log({ err, key, bucket }, 'error inserting file') + return res.send(500) + } else { + return res.send(200) + } + }) + }, - insertFile(req, res){ - metrics.inc("insertFile"); - const {key, bucket} = req; - logger.log({key, bucket}, "receiving request to insert file"); - return FileHandler.insertFile(bucket, key, req, function(err){ - if (err != null) { - logger.log({err, key, bucket}, "error inserting file"); - return res.send(500); - } else { - return res.send(200); - } - }); - }, + copyFile(req, res) { + metrics.inc('copyFile') + const { key, bucket } = req + const oldProject_id = req.body.source.project_id + const oldFile_id = req.body.source.file_id + logger.log( + { key, bucket, oldProject_id, oldFile_id }, + 'receiving request to copy file' + ) + return PersistorManager.copyFile( + bucket, + `${oldProject_id}/${oldFile_id}`, + key, + function(err) { + if (err != null) { + if (err instanceof Errors.NotFoundError) { + return res.send(404) + } else { + logger.log( + { err, oldProject_id, oldFile_id }, + 'something went wrong copying file' + ) + return res.send(500) + } + } else { + return res.send(200) + } + } + ) + }, - copyFile(req, res){ - metrics.inc("copyFile"); - const {key, bucket} = req; - const oldProject_id = req.body.source.project_id; - const oldFile_id = req.body.source.file_id; - logger.log({key, bucket, oldProject_id, oldFile_id}, "receiving request to copy file"); - return PersistorManager.copyFile(bucket, `${oldProject_id}/${oldFile_id}`, key, function(err){ - if (err != null) { - if (err instanceof Errors.NotFoundError) { - return res.send(404); - } else { - logger.log({err, oldProject_id, oldFile_id}, "something went wrong copying file"); - return res.send(500); - } - } else { - return res.send(200); - } - }); - }, + deleteFile(req, res) { + metrics.inc('deleteFile') + const { key, bucket } = req + logger.log({ key, bucket }, 'receiving request to delete file') + return FileHandler.deleteFile(bucket, key, function(err) { + if (err != null) { + logger.log({ err, key, bucket }, 'something went wrong deleting file') + return res.send(500) + } else { + return res.send(204) + } + }) + }, - deleteFile(req, res){ - metrics.inc("deleteFile"); - const {key, bucket} = req; - logger.log({key, bucket}, "receiving request to delete file"); - return FileHandler.deleteFile(bucket, key, function(err){ - if (err != null) { - logger.log({err, key, bucket}, "something went wrong deleting file"); - return res.send(500); - } else { - return res.send(204); - } - }); - }, + _get_range(header) { + const parsed = parseRange(maxSizeInBytes, header) + if (parsed === -1 || parsed === -2 || parsed.type !== 'bytes') { + return null + } else { + const range = parsed[0] + return { start: range.start, end: range.end } + } + }, - _get_range(header) { - const parsed = parseRange(maxSizeInBytes, header); - if ((parsed === -1) || (parsed === -2) || (parsed.type !== 'bytes')) { - return null; - } else { - const range = parsed[0]; - return {start: range.start, end: range.end}; - } - }, - - directorySize(req, res){ - metrics.inc("projectSize"); - const {project_id, bucket} = req; - logger.log({project_id, bucket}, "receiving request to project size"); - return FileHandler.getDirectorySize(bucket, project_id, function(err, size){ - if (err != null) { - logger.log({err, project_id, bucket}, "error inserting file"); - return res.send(500); - } else { - return res.json({'total bytes' : size}); - } - }); - } -}); + directorySize(req, res) { + metrics.inc('projectSize') + const { project_id, bucket } = req + logger.log({ project_id, bucket }, 'receiving request to project size') + return FileHandler.getDirectorySize(bucket, project_id, function( + err, + size + ) { + if (err != null) { + logger.log({ err, project_id, bucket }, 'error inserting file') + return res.send(500) + } else { + return res.json({ 'total bytes': size }) + } + }) + } +} diff --git a/services/filestore/app/js/FileConverter.js b/services/filestore/app/js/FileConverter.js index e241cd8275..04b599728a 100644 --- a/services/filestore/app/js/FileConverter.js +++ b/services/filestore/app/js/FileConverter.js @@ -9,77 +9,125 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const _ = require("underscore"); -const metrics = require("metrics-sharelatex"); -const logger = require("logger-sharelatex"); -const safe_exec = require("./SafeExec"); -const approvedFormats = ["png"]; -const Settings = require("settings-sharelatex"); +const _ = require('underscore') +const metrics = require('metrics-sharelatex') +const logger = require('logger-sharelatex') +const safe_exec = require('./SafeExec') +const approvedFormats = ['png'] +const Settings = require('settings-sharelatex') -const fourtySeconds = 40 * 1000; +const fourtySeconds = 40 * 1000 const childProcessOpts = { - killSignal: "SIGTERM", - timeout: fourtySeconds -}; - + killSignal: 'SIGTERM', + timeout: fourtySeconds +} module.exports = { + convert(sourcePath, requestedFormat, callback) { + logger.log({ sourcePath, requestedFormat }, 'converting file format') + const timer = new metrics.Timer('imageConvert') + const destPath = `${sourcePath}.${requestedFormat}` + sourcePath = `${sourcePath}[0]` + if (!_.include(approvedFormats, requestedFormat)) { + const err = new Error('invalid format requested') + return callback(err) + } + const width = '600x' + let command = [ + 'convert', + '-define', + `pdf:fit-page=${width}`, + '-flatten', + '-density', + '300', + sourcePath, + destPath + ] + command = Settings.commands.convertCommandPrefix.concat(command) + return safe_exec(command, childProcessOpts, function(err, stdout, stderr) { + timer.done() + if (err != null) { + logger.err( + { err, stderr, sourcePath, requestedFormat, destPath }, + 'something went wrong converting file' + ) + } else { + logger.log( + { sourcePath, requestedFormat, destPath }, + 'finished converting file' + ) + } + return callback(err, destPath) + }) + }, - convert(sourcePath, requestedFormat, callback){ - logger.log({sourcePath, requestedFormat}, "converting file format"); - const timer = new metrics.Timer("imageConvert"); - const destPath = `${sourcePath}.${requestedFormat}`; - sourcePath = `${sourcePath}[0]`; - if (!_.include(approvedFormats, requestedFormat)) { - const err = new Error("invalid format requested"); - return callback(err); - } - const width = "600x"; - let command = ["convert", "-define", `pdf:fit-page=${width}`, "-flatten", "-density", "300", sourcePath, destPath]; - command = Settings.commands.convertCommandPrefix.concat(command); - return safe_exec(command, childProcessOpts, function(err, stdout, stderr){ - timer.done(); - if (err != null) { - logger.err({err, stderr, sourcePath, requestedFormat, destPath}, "something went wrong converting file"); - } else { - logger.log({sourcePath, requestedFormat, destPath}, "finished converting file"); - } - return callback(err, destPath); - }); - }, + thumbnail(sourcePath, callback) { + const destPath = `${sourcePath}.png` + sourcePath = `${sourcePath}[0]` + const width = '260x' + let command = [ + 'convert', + '-flatten', + '-background', + 'white', + '-density', + '300', + '-define', + `pdf:fit-page=${width}`, + sourcePath, + '-resize', + width, + destPath + ] + logger.log({ sourcePath, destPath, command }, 'thumbnail convert file') + command = Settings.commands.convertCommandPrefix.concat(command) + return safe_exec(command, childProcessOpts, function(err, stdout, stderr) { + if (err != null) { + logger.err( + { err, stderr, sourcePath }, + 'something went wrong converting file to thumbnail' + ) + } else { + logger.log({ sourcePath, destPath }, 'finished thumbnailing file') + } + return callback(err, destPath) + }) + }, - thumbnail(sourcePath, callback){ - const destPath = `${sourcePath}.png`; - sourcePath = `${sourcePath}[0]`; - const width = "260x"; - let command = ["convert", "-flatten", "-background", "white", "-density", "300", "-define", `pdf:fit-page=${width}`, sourcePath, "-resize", width, destPath]; - logger.log({sourcePath, destPath, command}, "thumbnail convert file"); - command = Settings.commands.convertCommandPrefix.concat(command); - return safe_exec(command, childProcessOpts, function(err, stdout, stderr){ - if (err != null) { - logger.err({err, stderr, sourcePath}, "something went wrong converting file to thumbnail"); - } else { - logger.log({sourcePath, destPath}, "finished thumbnailing file"); - } - return callback(err, destPath); - }); - }, - - preview(sourcePath, callback){ - logger.log({sourcePath}, "preview convert file"); - const destPath = `${sourcePath}.png`; - sourcePath = `${sourcePath}[0]`; - const width = "548x"; - let command = ["convert", "-flatten", "-background", "white", "-density", "300", "-define", `pdf:fit-page=${width}`, sourcePath, "-resize", width, destPath]; - command = Settings.commands.convertCommandPrefix.concat(command); - return safe_exec(command, childProcessOpts, function(err, stdout, stderr){ - if (err != null) { - logger.err({err, stderr, sourcePath, destPath}, "something went wrong converting file to preview"); - } else { - logger.log({sourcePath, destPath}, "finished converting file to preview"); - } - return callback(err, destPath); - }); - } -}; + preview(sourcePath, callback) { + logger.log({ sourcePath }, 'preview convert file') + const destPath = `${sourcePath}.png` + sourcePath = `${sourcePath}[0]` + const width = '548x' + let command = [ + 'convert', + '-flatten', + '-background', + 'white', + '-density', + '300', + '-define', + `pdf:fit-page=${width}`, + sourcePath, + '-resize', + width, + destPath + ] + command = Settings.commands.convertCommandPrefix.concat(command) + return safe_exec(command, childProcessOpts, function(err, stdout, stderr) { + if (err != null) { + logger.err( + { err, stderr, sourcePath, destPath }, + 'something went wrong converting file to preview' + ) + } else { + logger.log( + { sourcePath, destPath }, + 'finished converting file to preview' + ) + } + return callback(err, destPath) + }) + } +} diff --git a/services/filestore/app/js/FileHandler.js b/services/filestore/app/js/FileHandler.js index c22a276b65..e63c813167 100644 --- a/services/filestore/app/js/FileHandler.js +++ b/services/filestore/app/js/FileHandler.js @@ -11,166 +11,225 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -let FileHandler; -const settings = require("settings-sharelatex"); -const PersistorManager = require("./PersistorManager"); -const LocalFileWriter = require("./LocalFileWriter"); -const logger = require("logger-sharelatex"); -const FileConverter = require("./FileConverter"); -const KeyBuilder = require("./KeyBuilder"); -const async = require("async"); -const ImageOptimiser = require("./ImageOptimiser"); -const Errors = require('./Errors'); +let FileHandler +const settings = require('settings-sharelatex') +const PersistorManager = require('./PersistorManager') +const LocalFileWriter = require('./LocalFileWriter') +const logger = require('logger-sharelatex') +const FileConverter = require('./FileConverter') +const KeyBuilder = require('./KeyBuilder') +const async = require('async') +const ImageOptimiser = require('./ImageOptimiser') +const Errors = require('./Errors') -module.exports = (FileHandler = { +module.exports = FileHandler = { + insertFile(bucket, key, stream, callback) { + const convertedKey = KeyBuilder.getConvertedFolderKey(key) + return PersistorManager.deleteDirectory(bucket, convertedKey, function( + error + ) { + if (error != null) { + return callback(error) + } + return PersistorManager.sendStream(bucket, key, stream, callback) + }) + }, - insertFile(bucket, key, stream, callback){ - const convertedKey = KeyBuilder.getConvertedFolderKey(key); - return PersistorManager.deleteDirectory(bucket, convertedKey, function(error) { - if (error != null) { return callback(error); } - return PersistorManager.sendStream(bucket, key, stream, callback); - }); - }, + deleteFile(bucket, key, callback) { + const convertedKey = KeyBuilder.getConvertedFolderKey(key) + return async.parallel( + [ + done => PersistorManager.deleteFile(bucket, key, done), + done => PersistorManager.deleteDirectory(bucket, convertedKey, done) + ], + callback + ) + }, - deleteFile(bucket, key, callback){ - const convertedKey = KeyBuilder.getConvertedFolderKey(key); - return async.parallel([ - done => PersistorManager.deleteFile(bucket, key, done), - done => PersistorManager.deleteDirectory(bucket, convertedKey, done) - ], callback); - }, + getFile(bucket, key, opts, callback) { + // In this call, opts can contain credentials + if (opts == null) { + opts = {} + } + logger.log({ bucket, key, opts: this._scrubSecrets(opts) }, 'getting file') + if (opts.format == null && opts.style == null) { + return this._getStandardFile(bucket, key, opts, callback) + } else { + return this._getConvertedFile(bucket, key, opts, callback) + } + }, - getFile(bucket, key, opts, callback){ - // In this call, opts can contain credentials - if (opts == null) { opts = {}; } - logger.log({bucket, key, opts:this._scrubSecrets(opts)}, "getting file"); - if ((opts.format == null) && (opts.style == null)) { - return this._getStandardFile(bucket, key, opts, callback); - } else { - return this._getConvertedFile(bucket, key, opts, callback); - } - }, + getFileSize(bucket, key, callback) { + return PersistorManager.getFileSize(bucket, key, callback) + }, - getFileSize(bucket, key, callback) { - return PersistorManager.getFileSize(bucket, key, callback); - }, + _getStandardFile(bucket, key, opts, callback) { + return PersistorManager.getFileStream(bucket, key, opts, function( + err, + fileStream + ) { + if (err != null && !(err instanceof Errors.NotFoundError)) { + logger.err( + { bucket, key, opts: FileHandler._scrubSecrets(opts) }, + 'error getting fileStream' + ) + } + return callback(err, fileStream) + }) + }, - _getStandardFile(bucket, key, opts, callback){ - return PersistorManager.getFileStream(bucket, key, opts, function(err, fileStream){ - if ((err != null) && !(err instanceof Errors.NotFoundError)) { - logger.err({bucket, key, opts:FileHandler._scrubSecrets(opts)}, "error getting fileStream"); - } - return callback(err, fileStream); - }); - }, + _getConvertedFile(bucket, key, opts, callback) { + const convertedKey = KeyBuilder.addCachingToKey(key, opts) + return PersistorManager.checkIfFileExists( + bucket, + convertedKey, + (err, exists) => { + if (err != null) { + return callback(err) + } + if (exists) { + return PersistorManager.getFileStream( + bucket, + convertedKey, + opts, + callback + ) + } else { + return this._getConvertedFileAndCache( + bucket, + key, + convertedKey, + opts, + callback + ) + } + } + ) + }, - _getConvertedFile(bucket, key, opts, callback){ - const convertedKey = KeyBuilder.addCachingToKey(key, opts); - return PersistorManager.checkIfFileExists(bucket, convertedKey, (err, exists)=> { - if (err != null) { - return callback(err); - } - if (exists) { - return PersistorManager.getFileStream(bucket, convertedKey, opts, callback); - } else { - return this._getConvertedFileAndCache(bucket, key, convertedKey, opts, callback); - } - }); - }, + _getConvertedFileAndCache(bucket, key, convertedKey, opts, callback) { + let convertedFsPath = '' + const originalFsPath = '' + return async.series( + [ + cb => { + return this._convertFile(bucket, key, opts, function( + err, + fileSystemPath, + originalFsPath + ) { + convertedFsPath = fileSystemPath + originalFsPath = originalFsPath + return cb(err) + }) + }, + cb => ImageOptimiser.compressPng(convertedFsPath, cb), + cb => + PersistorManager.sendFile(bucket, convertedKey, convertedFsPath, cb) + ], + function(err) { + if (err != null) { + LocalFileWriter.deleteFile(convertedFsPath, function() {}) + LocalFileWriter.deleteFile(originalFsPath, function() {}) + return callback(err) + } + // Send back the converted file from the local copy to avoid problems + // with the file not being present in S3 yet. As described in the + // documentation below, we have already made a 'HEAD' request in + // checkIfFileExists so we only have "eventual consistency" if we try + // to stream it from S3 here. This was a cause of many 403 errors. + // + // "Amazon S3 provides read-after-write consistency for PUTS of new + // objects in your S3 bucket in all regions with one caveat. The + // caveat is that if you make a HEAD or GET request to the key name + // (to find if the object exists) before creating the object, Amazon + // S3 provides eventual consistency for read-after-write."" + // https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel + return LocalFileWriter.getStream(convertedFsPath, function( + err, + readStream + ) { + if (err != null) { + return callback(err) + } + readStream.on('end', function() { + logger.log({ convertedFsPath }, 'deleting temporary file') + return LocalFileWriter.deleteFile(convertedFsPath, function() {}) + }) + return callback(null, readStream) + }) + } + ) + }, - _getConvertedFileAndCache(bucket, key, convertedKey, opts, callback){ - let convertedFsPath = ""; - const originalFsPath = ""; - return async.series([ - cb => { - return this._convertFile(bucket, key, opts, function(err, fileSystemPath, originalFsPath) { - convertedFsPath = fileSystemPath; - originalFsPath = originalFsPath; - return cb(err); - }); - }, - cb => ImageOptimiser.compressPng(convertedFsPath, cb), - cb => PersistorManager.sendFile(bucket, convertedKey, convertedFsPath, cb) - ], function(err){ - if (err != null) { - LocalFileWriter.deleteFile(convertedFsPath, function() {}); - LocalFileWriter.deleteFile(originalFsPath, function() {}); - return callback(err); - } - // Send back the converted file from the local copy to avoid problems - // with the file not being present in S3 yet. As described in the - // documentation below, we have already made a 'HEAD' request in - // checkIfFileExists so we only have "eventual consistency" if we try - // to stream it from S3 here. This was a cause of many 403 errors. - // - // "Amazon S3 provides read-after-write consistency for PUTS of new - // objects in your S3 bucket in all regions with one caveat. The - // caveat is that if you make a HEAD or GET request to the key name - // (to find if the object exists) before creating the object, Amazon - // S3 provides eventual consistency for read-after-write."" - // https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel - return LocalFileWriter.getStream(convertedFsPath, function(err, readStream) { - if (err != null) { return callback(err); } - readStream.on('end', function() { - logger.log({convertedFsPath}, "deleting temporary file"); - return LocalFileWriter.deleteFile(convertedFsPath, function() {}); - }); - return callback(null, readStream); - }); - }); - }, + _convertFile(bucket, originalKey, opts, callback) { + return this._writeS3FileToDisk(bucket, originalKey, opts, function( + err, + originalFsPath + ) { + if (err != null) { + return callback(err) + } + const done = function(err, destPath) { + if (err != null) { + logger.err( + { err, bucket, originalKey, opts: FileHandler._scrubSecrets(opts) }, + 'error converting file' + ) + return callback(err) + } + LocalFileWriter.deleteFile(originalFsPath, function() {}) + return callback(err, destPath, originalFsPath) + } - _convertFile(bucket, originalKey, opts, callback){ - return this._writeS3FileToDisk(bucket, originalKey, opts, function(err, originalFsPath){ - if (err != null) { - return callback(err); - } - const done = function(err, destPath){ - if (err != null) { - logger.err({err, bucket, originalKey, opts:FileHandler._scrubSecrets(opts)}, "error converting file"); - return callback(err); - } - LocalFileWriter.deleteFile(originalFsPath, function() {}); - return callback(err, destPath, originalFsPath); - }; + logger.log({ opts }, 'converting file depending on opts') - logger.log({opts}, "converting file depending on opts"); - - if (opts.format != null) { - return FileConverter.convert(originalFsPath, opts.format, done); - } else if (opts.style === "thumbnail") { - return FileConverter.thumbnail(originalFsPath, done); - } else if (opts.style === "preview") { - return FileConverter.preview(originalFsPath, done); - } else { - return callback(new Error(`should have specified opts to convert file with ${JSON.stringify(opts)}`)); - } - }); - }, + if (opts.format != null) { + return FileConverter.convert(originalFsPath, opts.format, done) + } else if (opts.style === 'thumbnail') { + return FileConverter.thumbnail(originalFsPath, done) + } else if (opts.style === 'preview') { + return FileConverter.preview(originalFsPath, done) + } else { + return callback( + new Error( + `should have specified opts to convert file with ${JSON.stringify( + opts + )}` + ) + ) + } + }) + }, + _writeS3FileToDisk(bucket, key, opts, callback) { + return PersistorManager.getFileStream(bucket, key, opts, function( + err, + fileStream + ) { + if (err != null) { + return callback(err) + } + return LocalFileWriter.writeStream(fileStream, key, callback) + }) + }, - _writeS3FileToDisk(bucket, key, opts, callback){ - return PersistorManager.getFileStream(bucket, key, opts, function(err, fileStream){ - if (err != null) { - return callback(err); - } - return LocalFileWriter.writeStream(fileStream, key, callback); - }); - }, + getDirectorySize(bucket, project_id, callback) { + logger.log({ bucket, project_id }, 'getting project size') + return PersistorManager.directorySize(bucket, project_id, function( + err, + size + ) { + if (err != null) { + logger.err({ bucket, project_id }, 'error getting size') + } + return callback(err, size) + }) + }, - getDirectorySize(bucket, project_id, callback){ - logger.log({bucket, project_id}, "getting project size"); - return PersistorManager.directorySize(bucket, project_id, function(err, size){ - if (err != null) { - logger.err({bucket, project_id}, "error getting size"); - } - return callback(err, size); - }); - }, - - _scrubSecrets(opts){ - const safe = Object.assign({}, opts); - delete safe.credentials; - return safe; - } -}); + _scrubSecrets(opts) { + const safe = Object.assign({}, opts) + delete safe.credentials + return safe + } +} diff --git a/services/filestore/app/js/HealthCheckController.js b/services/filestore/app/js/HealthCheckController.js index e360891609..eecadb00d9 100644 --- a/services/filestore/app/js/HealthCheckController.js +++ b/services/filestore/app/js/HealthCheckController.js @@ -6,71 +6,75 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const fs = require("fs-extra"); -const path = require("path"); -const async = require("async"); -const fileConverter = require("./FileConverter"); -const keyBuilder = require("./KeyBuilder"); -const fileController = require("./FileController"); -const logger = require('logger-sharelatex'); -const settings = require("settings-sharelatex"); -const streamBuffers = require("stream-buffers"); -const _ = require('underscore'); +const fs = require('fs-extra') +const path = require('path') +const async = require('async') +const fileConverter = require('./FileConverter') +const keyBuilder = require('./KeyBuilder') +const fileController = require('./FileController') +const logger = require('logger-sharelatex') +const settings = require('settings-sharelatex') +const streamBuffers = require('stream-buffers') +const _ = require('underscore') +const checkCanStoreFiles = function(callback) { + callback = _.once(callback) + const req = { params: {}, query: {}, headers: {} } + req.params.project_id = settings.health_check.project_id + req.params.file_id = settings.health_check.file_id + const myWritableStreamBuffer = new streamBuffers.WritableStreamBuffer({ + initialSize: 100 + }) + const res = { + send(code) { + if (code !== 200) { + return callback(new Error(`non-200 code from getFile: ${code}`)) + } + } + } + myWritableStreamBuffer.send = res.send + return keyBuilder.userFileKey(req, res, function() { + fileController.getFile(req, myWritableStreamBuffer) + return myWritableStreamBuffer.on('close', function() { + if (myWritableStreamBuffer.size() > 0) { + return callback() + } else { + const err = 'no data in write stream buffer for health check' + logger.err({ err }, 'error performing health check') + return callback(err) + } + }) + }) +} -const checkCanStoreFiles = function(callback){ - callback = _.once(callback); - const req = {params:{}, query:{}, headers:{}}; - req.params.project_id = settings.health_check.project_id; - req.params.file_id = settings.health_check.file_id; - const myWritableStreamBuffer = new streamBuffers.WritableStreamBuffer({initialSize: 100}); - const res = { - send(code) { - if (code !== 200) { - return callback(new Error(`non-200 code from getFile: ${code}`)); - } - } - }; - myWritableStreamBuffer.send = res.send; - return keyBuilder.userFileKey(req, res, function() { - fileController.getFile(req, myWritableStreamBuffer); - return myWritableStreamBuffer.on("close", function() { - if (myWritableStreamBuffer.size() > 0) { - return callback(); - } else { - const err = "no data in write stream buffer for health check"; - logger.err({err,}, "error performing health check"); - return callback(err); - } - }); - }); -}; - -const checkFileConvert = function(callback){ - if (!settings.enableConversions) { - return callback(); - } - const imgPath = path.join(settings.path.uploadFolder, "/tiny.pdf"); - return async.waterfall([ - cb => fs.copy("./tiny.pdf", imgPath, cb), - cb => fileConverter.thumbnail(imgPath, cb), - (resultPath, cb) => fs.unlink(resultPath, cb), - cb => fs.unlink(imgPath, cb) - ], callback); -}; - +const checkFileConvert = function(callback) { + if (!settings.enableConversions) { + return callback() + } + const imgPath = path.join(settings.path.uploadFolder, '/tiny.pdf') + return async.waterfall( + [ + cb => fs.copy('./tiny.pdf', imgPath, cb), + cb => fileConverter.thumbnail(imgPath, cb), + (resultPath, cb) => fs.unlink(resultPath, cb), + cb => fs.unlink(imgPath, cb) + ], + callback + ) +} module.exports = { - - check(req, res) { - logger.log({}, "performing health check"); - return async.parallel([checkFileConvert, checkCanStoreFiles], function(err){ - if (err != null) { - logger.err({err}, "Health check: error running"); - return res.send(500); - } else { - return res.send(200); - } - }); - } -}; + check(req, res) { + logger.log({}, 'performing health check') + return async.parallel([checkFileConvert, checkCanStoreFiles], function( + err + ) { + if (err != null) { + logger.err({ err }, 'Health check: error running') + return res.send(500) + } else { + return res.send(200) + } + }) + } +} diff --git a/services/filestore/app/js/ImageOptimiser.js b/services/filestore/app/js/ImageOptimiser.js index 0126786167..99f1eb21f6 100644 --- a/services/filestore/app/js/ImageOptimiser.js +++ b/services/filestore/app/js/ImageOptimiser.js @@ -9,36 +9,36 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const { - exec -} = require('child_process'); -const logger = require("logger-sharelatex"); -const Settings = require("settings-sharelatex"); +const { exec } = require('child_process') +const logger = require('logger-sharelatex') +const Settings = require('settings-sharelatex') -module.exports = { - - compressPng(localPath, callback){ - const startTime = new Date(); - logger.log({localPath}, "optimising png path"); - const args = `optipng ${localPath}`; - const opts = { - timeout: 30 * 1000, - killSignal: "SIGKILL" - }; - if (!Settings.enableConversions) { - const error = new Error("Image conversions are disabled"); - return callback(error); - } - return exec(args, opts,function(err, stdout, stderr){ - if ((err != null) && (err.signal === 'SIGKILL')) { - logger.warn({err, stderr, localPath}, "optimiser timeout reached"); - err = null; - } else if (err != null) { - logger.err({err, stderr, localPath}, "something went wrong converting compressPng"); - } else { - logger.log({localPath}, "finished compressPng file"); - } - return callback(err); - }); - } -}; +module.exports = { + compressPng(localPath, callback) { + const startTime = new Date() + logger.log({ localPath }, 'optimising png path') + const args = `optipng ${localPath}` + const opts = { + timeout: 30 * 1000, + killSignal: 'SIGKILL' + } + if (!Settings.enableConversions) { + const error = new Error('Image conversions are disabled') + return callback(error) + } + return exec(args, opts, function(err, stdout, stderr) { + if (err != null && err.signal === 'SIGKILL') { + logger.warn({ err, stderr, localPath }, 'optimiser timeout reached') + err = null + } else if (err != null) { + logger.err( + { err, stderr, localPath }, + 'something went wrong converting compressPng' + ) + } else { + logger.log({ localPath }, 'finished compressPng file') + } + return callback(err) + }) + } +} diff --git a/services/filestore/app/js/KeyBuilder.js b/services/filestore/app/js/KeyBuilder.js index 5d3a36e6b1..110900c991 100644 --- a/services/filestore/app/js/KeyBuilder.js +++ b/services/filestore/app/js/KeyBuilder.js @@ -11,65 +11,61 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const settings = require("settings-sharelatex"); +const settings = require('settings-sharelatex') module.exports = { + getConvertedFolderKey(key) { + return (key = `${key}-converted-cache/`) + }, + addCachingToKey(key, opts) { + key = this.getConvertedFolderKey(key) + if (opts.format != null && opts.style == null) { + key = `${key}format-${opts.format}` + } + if (opts.style != null && opts.format == null) { + key = `${key}style-${opts.style}` + } + if (opts.style != null && opts.format != null) { + key = `${key}format-${opts.format}-style-${opts.style}` + } + return key + }, - getConvertedFolderKey(key){ - return key = `${key}-converted-cache/`; - }, + userFileKey(req, res, next) { + const { project_id, file_id } = req.params + req.key = `${project_id}/${file_id}` + req.bucket = settings.filestore.stores.user_files + return next() + }, - addCachingToKey(key, opts){ - key = this.getConvertedFolderKey(key); - if ((opts.format != null) && (opts.style == null)) { - key = `${key}format-${opts.format}`; - } - if ((opts.style != null) && (opts.format == null)) { - key = `${key}style-${opts.style}`; - } - if ((opts.style != null) && (opts.format != null)) { - key = `${key}format-${opts.format}-style-${opts.style}`; - } - return key; - }, + publicFileKey(req, res, next) { + const { project_id, public_file_id } = req.params + if (settings.filestore.stores.public_files == null) { + return res.status(501).send('public files not available') + } else { + req.key = `${project_id}/${public_file_id}` + req.bucket = settings.filestore.stores.public_files + return next() + } + }, + templateFileKey(req, res, next) { + const { template_id, format, version, sub_type } = req.params + req.key = `${template_id}/v/${version}/${format}` + if (sub_type != null) { + req.key = `${req.key}/${sub_type}` + } + req.bucket = settings.filestore.stores.template_files + req.version = version + const opts = req.query + return next() + }, - userFileKey(req, res, next){ - const {project_id, file_id} = req.params; - req.key = `${project_id}/${file_id}`; - req.bucket = settings.filestore.stores.user_files; - return next(); - }, - - publicFileKey(req, res, next){ - const {project_id, public_file_id} = req.params; - if ((settings.filestore.stores.public_files == null)) { - return res.status(501).send("public files not available"); - } else { - req.key = `${project_id}/${public_file_id}`; - req.bucket = settings.filestore.stores.public_files; - return next(); - } - }, - - templateFileKey(req, res, next){ - const {template_id, format, version, sub_type} = req.params; - req.key = `${template_id}/v/${version}/${format}`; - if (sub_type != null) { - req.key = `${req.key}/${sub_type}`; - } - req.bucket = settings.filestore.stores.template_files; - req.version = version; - const opts = req.query; - return next(); - }, - - publicProjectKey(req, res, next){ - const {project_id} = req.params; - req.project_id = project_id; - req.bucket = settings.filestore.stores.user_files; - return next(); - } -}; - + publicProjectKey(req, res, next) { + const { project_id } = req.params + req.project_id = project_id + req.bucket = settings.filestore.stores.user_files + return next() + } +} diff --git a/services/filestore/app/js/LocalFileWriter.js b/services/filestore/app/js/LocalFileWriter.js index b0bfedcc04..8a541a35e9 100644 --- a/services/filestore/app/js/LocalFileWriter.js +++ b/services/filestore/app/js/LocalFileWriter.js @@ -9,73 +9,83 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const fs = require("fs"); -const uuid = require('node-uuid'); -const path = require("path"); -const _ = require("underscore"); -const logger = require("logger-sharelatex"); -const metrics = require("metrics-sharelatex"); -const Settings = require("settings-sharelatex"); -const Errors = require("./Errors"); +const fs = require('fs') +const uuid = require('node-uuid') +const path = require('path') +const _ = require('underscore') +const logger = require('logger-sharelatex') +const metrics = require('metrics-sharelatex') +const Settings = require('settings-sharelatex') +const Errors = require('./Errors') -module.exports = { +module.exports = { + writeStream(stream, key, callback) { + const timer = new metrics.Timer('writingFile') + callback = _.once(callback) + const fsPath = this._getPath(key) + logger.log({ fsPath }, 'writing file locally') + const writeStream = fs.createWriteStream(fsPath) + writeStream.on('finish', function() { + timer.done() + logger.log({ fsPath }, 'finished writing file locally') + return callback(null, fsPath) + }) + writeStream.on('error', function(err) { + logger.err( + { err, fsPath }, + 'problem writing file locally, with write stream' + ) + return callback(err) + }) + stream.on('error', function(err) { + logger.log( + { err, fsPath }, + 'problem writing file locally, with read stream' + ) + return callback(err) + }) + return stream.pipe(writeStream) + }, - writeStream(stream, key, callback){ - const timer = new metrics.Timer("writingFile"); - callback = _.once(callback); - const fsPath = this._getPath(key); - logger.log({fsPath}, "writing file locally"); - const writeStream = fs.createWriteStream(fsPath); - writeStream.on("finish", function() { - timer.done(); - logger.log({fsPath}, "finished writing file locally"); - return callback(null, fsPath); - }); - writeStream.on("error", function(err){ - logger.err({err, fsPath}, "problem writing file locally, with write stream"); - return callback(err); - }); - stream.on("error", function(err){ - logger.log({err, fsPath}, "problem writing file locally, with read stream"); - return callback(err); - }); - return stream.pipe(writeStream); - }, + getStream(fsPath, _callback) { + if (_callback == null) { + _callback = function(err, res) {} + } + const callback = _.once(_callback) + const timer = new metrics.Timer('readingFile') + logger.log({ fsPath }, 'reading file locally') + const readStream = fs.createReadStream(fsPath) + readStream.on('end', function() { + timer.done() + return logger.log({ fsPath }, 'finished reading file locally') + }) + readStream.on('error', function(err) { + logger.err( + { err, fsPath }, + 'problem reading file locally, with read stream' + ) + if (err.code === 'ENOENT') { + return callback(new Errors.NotFoundError(err.message), null) + } else { + return callback(err) + } + }) + return callback(null, readStream) + }, - getStream(fsPath, _callback) { - if (_callback == null) { _callback = function(err, res){}; } - const callback = _.once(_callback); - const timer = new metrics.Timer("readingFile"); - logger.log({fsPath}, "reading file locally"); - const readStream = fs.createReadStream(fsPath); - readStream.on("end", function() { - timer.done(); - return logger.log({fsPath}, "finished reading file locally"); - }); - readStream.on("error", function(err){ - logger.err({err, fsPath}, "problem reading file locally, with read stream"); - if (err.code === 'ENOENT') { - return callback(new Errors.NotFoundError(err.message), null); - } else { - return callback(err); - } - }); - return callback(null, readStream); - }, + deleteFile(fsPath, callback) { + if (fsPath == null || fsPath === '') { + return callback() + } + logger.log({ fsPath }, 'removing local temp file') + return fs.unlink(fsPath, callback) + }, - deleteFile(fsPath, callback){ - if ((fsPath == null) || (fsPath === "")) { - return callback(); - } - logger.log({fsPath}, "removing local temp file"); - return fs.unlink(fsPath, callback); - }, - - _getPath(key){ - if ((key == null)) { - key = uuid.v1(); - } - key = key.replace(/\//g,"-"); - return path.join(Settings.path.uploadFolder, key); - } -}; + _getPath(key) { + if (key == null) { + key = uuid.v1() + } + key = key.replace(/\//g, '-') + return path.join(Settings.path.uploadFolder, key) + } +} diff --git a/services/filestore/app/js/PersistorManager.js b/services/filestore/app/js/PersistorManager.js index 95a4122591..182e39b085 100644 --- a/services/filestore/app/js/PersistorManager.js +++ b/services/filestore/app/js/PersistorManager.js @@ -7,24 +7,46 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const settings = require("settings-sharelatex"); -const logger = require("logger-sharelatex"); +const settings = require('settings-sharelatex') +const logger = require('logger-sharelatex') // assume s3 if none specified -__guard__(settings != null ? settings.filestore : undefined, x => x.backend || (settings.filestore.backend = "s3")); +__guard__( + settings != null ? settings.filestore : undefined, + x => x.backend || (settings.filestore.backend = 's3') +) -logger.log({backend:__guard__(settings != null ? settings.filestore : undefined, x1 => x1.backend)}, "Loading backend"); -module.exports = (() => { switch (__guard__(settings != null ? settings.filestore : undefined, x2 => x2.backend)) { - case "aws-sdk": - return require("./AWSSDKPersistorManager"); - case "s3": - return require("./S3PersistorManager"); - case "fs": - return require("./FSPersistorManager"); - default: - throw new Error( `Unknown filestore backend: ${settings.filestore.backend}` ); -} })(); +logger.log( + { + backend: __guard__( + settings != null ? settings.filestore : undefined, + x1 => x1.backend + ) + }, + 'Loading backend' +) +module.exports = (() => { + switch ( + __guard__( + settings != null ? settings.filestore : undefined, + x2 => x2.backend + ) + ) { + case 'aws-sdk': + return require('./AWSSDKPersistorManager') + case 's3': + return require('./S3PersistorManager') + case 'fs': + return require('./FSPersistorManager') + default: + throw new Error( + `Unknown filestore backend: ${settings.filestore.backend}` + ) + } +})() function __guard__(value, transform) { - return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; -} \ No newline at end of file + return typeof value !== 'undefined' && value !== null + ? transform(value) + : undefined +} diff --git a/services/filestore/app/js/S3PersistorManager.js b/services/filestore/app/js/S3PersistorManager.js index 0face2de53..cadf38172a 100644 --- a/services/filestore/app/js/S3PersistorManager.js +++ b/services/filestore/app/js/S3PersistorManager.js @@ -19,282 +19,358 @@ // to use aws-sdk throughout, see the comments in AWSSDKPersistorManager for // details. The knox library is unmaintained and has bugs. -const http = require('http'); -http.globalAgent.maxSockets = 300; -const https = require('https'); -https.globalAgent.maxSockets = 300; -const settings = require("settings-sharelatex"); -const request = require("request"); -const logger = require("logger-sharelatex"); -const metrics = require("metrics-sharelatex"); -const fs = require("fs"); -const knox = require("knox"); -const path = require("path"); -const LocalFileWriter = require("./LocalFileWriter"); -const Errors = require("./Errors"); -const _ = require("underscore"); -const awsS3 = require("aws-sdk/clients/s3"); -const URL = require('url'); +const http = require('http') +http.globalAgent.maxSockets = 300 +const https = require('https') +https.globalAgent.maxSockets = 300 +const settings = require('settings-sharelatex') +const request = require('request') +const logger = require('logger-sharelatex') +const metrics = require('metrics-sharelatex') +const fs = require('fs') +const knox = require('knox') +const path = require('path') +const LocalFileWriter = require('./LocalFileWriter') +const Errors = require('./Errors') +const _ = require('underscore') +const awsS3 = require('aws-sdk/clients/s3') +const URL = require('url') -const thirtySeconds = 30 * 1000; +const thirtySeconds = 30 * 1000 -const buildDefaultOptions = function(bucketName, method, key){ - let endpoint; - if (settings.filestore.s3.endpoint) { - endpoint = `${settings.filestore.s3.endpoint}/${bucketName}`; - } else { - endpoint = `https://${bucketName}.s3.amazonaws.com`; - } - return { - aws: { - key: settings.filestore.s3.key, - secret: settings.filestore.s3.secret, - bucket: bucketName - }, - method, - timeout: thirtySeconds, - uri:`${endpoint}/${key}` - }; -}; +const buildDefaultOptions = function(bucketName, method, key) { + let endpoint + if (settings.filestore.s3.endpoint) { + endpoint = `${settings.filestore.s3.endpoint}/${bucketName}` + } else { + endpoint = `https://${bucketName}.s3.amazonaws.com` + } + return { + aws: { + key: settings.filestore.s3.key, + secret: settings.filestore.s3.secret, + bucket: bucketName + }, + method, + timeout: thirtySeconds, + uri: `${endpoint}/${key}` + } +} const getS3Options = function(credentials) { - const options = { - credentials: { - accessKeyId: credentials.auth_key, - secretAccessKey: credentials.auth_secret - } - }; + const options = { + credentials: { + accessKeyId: credentials.auth_key, + secretAccessKey: credentials.auth_secret + } + } - if (settings.filestore.s3.endpoint) { - const endpoint = URL.parse(settings.filestore.s3.endpoint); - options.endpoint = settings.filestore.s3.endpoint; - options.sslEnabled = endpoint.protocol === 'https'; - } + if (settings.filestore.s3.endpoint) { + const endpoint = URL.parse(settings.filestore.s3.endpoint) + options.endpoint = settings.filestore.s3.endpoint + options.sslEnabled = endpoint.protocol === 'https' + } - return options; -}; + return options +} -const defaultS3Client = new awsS3(getS3Options({ - auth_key: settings.filestore.s3.key, - auth_secret: settings.filestore.s3.secret -})); +const defaultS3Client = new awsS3( + getS3Options({ + auth_key: settings.filestore.s3.key, + auth_secret: settings.filestore.s3.secret + }) +) const getS3Client = function(credentials) { - if (credentials != null) { - return new awsS3(getS3Options(credentials)); - } else { - return defaultS3Client; - } -}; + if (credentials != null) { + return new awsS3(getS3Options(credentials)) + } else { + return defaultS3Client + } +} const getKnoxClient = bucketName => { - const options = { - key: settings.filestore.s3.key, - secret: settings.filestore.s3.secret, - bucket: bucketName - }; - if (settings.filestore.s3.endpoint) { - const endpoint = URL.parse(settings.filestore.s3.endpoint); - options.endpoint = endpoint.hostname; - options.port = endpoint.port; - } - return knox.createClient(options); -}; + const options = { + key: settings.filestore.s3.key, + secret: settings.filestore.s3.secret, + bucket: bucketName + } + if (settings.filestore.s3.endpoint) { + const endpoint = URL.parse(settings.filestore.s3.endpoint) + options.endpoint = endpoint.hostname + options.port = endpoint.port + } + return knox.createClient(options) +} module.exports = { + sendFile(bucketName, key, fsPath, callback) { + const s3Client = getKnoxClient(bucketName) + let uploaded = 0 + const putEventEmiter = s3Client.putFile(fsPath, key, function(err, res) { + metrics.count('s3.egress', uploaded) + if (err != null) { + logger.err( + { err, bucketName, key, fsPath }, + 'something went wrong uploading file to s3' + ) + return callback(err) + } + if (res == null) { + logger.err( + { err, res, bucketName, key, fsPath }, + 'no response from s3 put file' + ) + return callback('no response from put file') + } + if (res.statusCode !== 200) { + logger.err( + { bucketName, key, fsPath }, + 'non 200 response from s3 putting file' + ) + return callback('non 200 response from s3 on put file') + } + logger.log({ res, bucketName, key, fsPath }, 'file uploaded to s3') + return callback(err) + }) + putEventEmiter.on('error', function(err) { + logger.err( + { err, bucketName, key, fsPath }, + 'error emmited on put of file' + ) + return callback(err) + }) + return putEventEmiter.on( + 'progress', + progress => (uploaded = progress.written) + ) + }, - sendFile(bucketName, key, fsPath, callback){ - const s3Client = getKnoxClient(bucketName); - let uploaded = 0; - const putEventEmiter = s3Client.putFile(fsPath, key, function(err, res){ - metrics.count('s3.egress', uploaded); - if (err != null) { - logger.err({err, bucketName, key, fsPath},"something went wrong uploading file to s3"); - return callback(err); - } - if ((res == null)) { - logger.err({err, res, bucketName, key, fsPath}, "no response from s3 put file"); - return callback("no response from put file"); - } - if (res.statusCode !== 200) { - logger.err({bucketName, key, fsPath}, "non 200 response from s3 putting file"); - return callback("non 200 response from s3 on put file"); - } - logger.log({res, bucketName, key, fsPath},"file uploaded to s3"); - return callback(err); - }); - putEventEmiter.on("error", function(err){ - logger.err({err, bucketName, key, fsPath}, "error emmited on put of file"); - return callback(err); - }); - return putEventEmiter.on("progress", progress => uploaded = progress.written); - }, + sendStream(bucketName, key, readStream, callback) { + logger.log({ bucketName, key }, 'sending file to s3') + readStream.on('error', err => + logger.err({ bucketName, key }, 'error on stream to send to s3') + ) + return LocalFileWriter.writeStream(readStream, null, (err, fsPath) => { + if (err != null) { + logger.err( + { bucketName, key, fsPath, err }, + 'something went wrong writing stream to disk' + ) + return callback(err) + } + return this.sendFile(bucketName, key, fsPath, ( + err // delete the temporary file created above and return the original error + ) => LocalFileWriter.deleteFile(fsPath, () => callback(err))) + }) + }, - sendStream(bucketName, key, readStream, callback){ - logger.log({bucketName, key}, "sending file to s3"); - readStream.on("error", err => logger.err({bucketName, key}, "error on stream to send to s3")); - return LocalFileWriter.writeStream(readStream, null, (err, fsPath)=> { - if (err != null) { - logger.err({bucketName, key, fsPath, err}, "something went wrong writing stream to disk"); - return callback(err); - } - return this.sendFile(bucketName, key, fsPath, err => // delete the temporary file created above and return the original error - LocalFileWriter.deleteFile(fsPath, () => callback(err))); - }); - }, + // opts may be {start: Number, end: Number} + getFileStream(bucketName, key, opts, callback) { + if (callback == null) { + callback = function(err, res) {} + } + opts = opts || {} + callback = _.once(callback) + logger.log({ bucketName, key }, 'getting file from s3') - // opts may be {start: Number, end: Number} - getFileStream(bucketName, key, opts, callback){ - if (callback == null) { callback = function(err, res){}; } - opts = opts || {}; - callback = _.once(callback); - logger.log({bucketName, key}, "getting file from s3"); + const s3 = getS3Client(opts.credentials) + const s3Params = { + Bucket: bucketName, + Key: key + } + if (opts.start != null && opts.end != null) { + s3Params.Range = `bytes=${opts.start}-${opts.end}` + } + const s3Request = s3.getObject(s3Params) - const s3 = getS3Client(opts.credentials); - const s3Params = { - Bucket: bucketName, - Key: key - }; - if ((opts.start != null) && (opts.end != null)) { - s3Params.Range = `bytes=${opts.start}-${opts.end}`; - } - const s3Request = s3.getObject(s3Params); + s3Request.on( + 'httpHeaders', + (statusCode, headers, response, statusMessage) => { + if ([403, 404].includes(statusCode)) { + // S3 returns a 403 instead of a 404 when the user doesn't have + // permission to list the bucket contents. + logger.log({ bucketName, key }, 'file not found in s3') + return callback( + new Errors.NotFoundError( + `File not found in S3: ${bucketName}:${key}` + ), + null + ) + } + if (![200, 206].includes(statusCode)) { + logger.log( + { bucketName, key }, + `error getting file from s3: ${statusCode}` + ) + return callback( + new Error( + `Got non-200 response from S3: ${statusCode} ${statusMessage}` + ), + null + ) + } + const stream = response.httpResponse.createUnbufferedStream() + stream.on('data', data => metrics.count('s3.ingress', data.byteLength)) - s3Request.on('httpHeaders', (statusCode, headers, response, statusMessage) => { - if ([403, 404].includes(statusCode)) { - // S3 returns a 403 instead of a 404 when the user doesn't have - // permission to list the bucket contents. - logger.log({ bucketName, key }, "file not found in s3"); - return callback(new Errors.NotFoundError(`File not found in S3: ${bucketName}:${key}`), null); - } - if (![200, 206].includes(statusCode)) { - logger.log({bucketName, key }, `error getting file from s3: ${statusCode}`); - return callback(new Error(`Got non-200 response from S3: ${statusCode} ${statusMessage}`), null); - } - const stream = response.httpResponse.createUnbufferedStream(); - stream.on('data', data => metrics.count('s3.ingress', data.byteLength)); + return callback(null, stream) + } + ) - return callback(null, stream); - }); + s3Request.on('error', err => { + logger.err({ err, bucketName, key }, 'error getting file stream from s3') + return callback(err) + }) - s3Request.on('error', err => { - logger.err({ err, bucketName, key }, "error getting file stream from s3"); - return callback(err); - }); + return s3Request.send() + }, - return s3Request.send(); - }, + getFileSize(bucketName, key, callback) { + logger.log({ bucketName, key }, 'getting file size from S3') + const s3 = getS3Client() + return s3.headObject({ Bucket: bucketName, Key: key }, function(err, data) { + if (err != null) { + if ([403, 404].includes(err.statusCode)) { + // S3 returns a 403 instead of a 404 when the user doesn't have + // permission to list the bucket contents. + logger.log( + { + bucketName, + key + }, + 'file not found in s3' + ) + callback( + new Errors.NotFoundError( + `File not found in S3: ${bucketName}:${key}` + ) + ) + } else { + logger.err( + { + bucketName, + key, + err + }, + 'error performing S3 HeadObject' + ) + callback(err) + } + return + } + return callback(null, data.ContentLength) + }) + }, - getFileSize(bucketName, key, callback) { - logger.log({ bucketName, key }, "getting file size from S3"); - const s3 = getS3Client(); - return s3.headObject({ Bucket: bucketName, Key: key }, function(err, data) { - if (err != null) { - if ([403, 404].includes(err.statusCode)) { - // S3 returns a 403 instead of a 404 when the user doesn't have - // permission to list the bucket contents. - logger.log({ - bucketName, - key - }, "file not found in s3"); - callback( - new Errors.NotFoundError(`File not found in S3: ${bucketName}:${key}`) - ); - } else { - logger.err({ - bucketName, - key, - err - }, "error performing S3 HeadObject"); - callback(err); - } - return; - } - return callback(null, data.ContentLength); - }); - }, + copyFile(bucketName, sourceKey, destKey, callback) { + logger.log({ bucketName, sourceKey, destKey }, 'copying file in s3') + const source = bucketName + '/' + sourceKey + // use the AWS SDK instead of knox due to problems with error handling (https://github.com/Automattic/knox/issues/114) + const s3 = getS3Client() + return s3.copyObject( + { Bucket: bucketName, Key: destKey, CopySource: source }, + function(err) { + if (err != null) { + if (err.code === 'NoSuchKey') { + logger.err( + { bucketName, sourceKey }, + 'original file not found in s3 when copying' + ) + return callback( + new Errors.NotFoundError( + 'original file not found in S3 when copying' + ) + ) + } else { + logger.err( + { err, bucketName, sourceKey, destKey }, + 'something went wrong copying file in aws' + ) + return callback(err) + } + } else { + return callback() + } + } + ) + }, - copyFile(bucketName, sourceKey, destKey, callback){ - logger.log({bucketName, sourceKey, destKey}, "copying file in s3"); - const source = bucketName + '/' + sourceKey; - // use the AWS SDK instead of knox due to problems with error handling (https://github.com/Automattic/knox/issues/114) - const s3 = getS3Client(); - return s3.copyObject({Bucket: bucketName, Key: destKey, CopySource: source}, function(err) { - if (err != null) { - if (err.code === 'NoSuchKey') { - logger.err({bucketName, sourceKey}, "original file not found in s3 when copying"); - return callback(new Errors.NotFoundError("original file not found in S3 when copying")); - } else { - logger.err({err, bucketName, sourceKey, destKey}, "something went wrong copying file in aws"); - return callback(err); - } - } else { - return callback(); - } - }); - }, + deleteFile(bucketName, key, callback) { + logger.log({ bucketName, key }, 'delete file in s3') + const options = buildDefaultOptions(bucketName, 'delete', key) + return request(options, function(err, res) { + if (err != null) { + logger.err( + { err, res, bucketName, key }, + 'something went wrong deleting file in aws' + ) + } + return callback(err) + }) + }, - deleteFile(bucketName, key, callback){ - logger.log({bucketName, key}, "delete file in s3"); - const options = buildDefaultOptions(bucketName, "delete", key); - return request(options, function(err, res){ - if (err != null) { - logger.err({err, res, bucketName, key}, "something went wrong deleting file in aws"); - } - return callback(err); - }); - }, + deleteDirectory(bucketName, key, _callback) { + // deleteMultiple can call the callback multiple times so protect against this. + const callback = function(...args) { + _callback(...Array.from(args || [])) + return (_callback = function() {}) + } - deleteDirectory(bucketName, key, _callback){ - // deleteMultiple can call the callback multiple times so protect against this. - const callback = function(...args) { - _callback(...Array.from(args || [])); - return _callback = function() {}; - }; + logger.log({ key, bucketName }, 'deleting directory') + const s3Client = getKnoxClient(bucketName) + return s3Client.list({ prefix: key }, function(err, data) { + if (err != null) { + logger.err( + { err, bucketName, key }, + 'something went wrong listing prefix in aws' + ) + return callback(err) + } + const keys = _.map(data.Contents, entry => entry.Key) + return s3Client.deleteMultiple(keys, callback) + }) + }, - logger.log({key, bucketName}, "deleting directory"); - const s3Client = getKnoxClient(bucketName); - return s3Client.list({prefix:key}, function(err, data){ - if (err != null) { - logger.err({err, bucketName, key}, "something went wrong listing prefix in aws"); - return callback(err); - } - const keys = _.map(data.Contents, entry => entry.Key); - return s3Client.deleteMultiple(keys, callback); - }); - }, + checkIfFileExists(bucketName, key, callback) { + logger.log({ bucketName, key }, 'checking if file exists in s3') + const options = buildDefaultOptions(bucketName, 'head', key) + return request(options, function(err, res) { + if (err != null) { + logger.err( + { err, res, bucketName, key }, + 'something went wrong checking file in aws' + ) + return callback(err) + } + if (res == null) { + logger.err( + { err, res, bucketName, key }, + 'no response object returned when checking if file exists' + ) + err = new Error(`no response from s3 ${bucketName} ${key}`) + return callback(err) + } + const exists = res.statusCode === 200 + logger.log({ bucketName, key, exists }, 'checked if file exsists in s3') + return callback(err, exists) + }) + }, - checkIfFileExists(bucketName, key, callback){ - logger.log({bucketName, key}, "checking if file exists in s3"); - const options = buildDefaultOptions(bucketName, "head", key); - return request(options, function(err, res){ - if (err != null) { - logger.err({err, res, bucketName, key}, "something went wrong checking file in aws"); - return callback(err); - } - if ((res == null)) { - logger.err({err, res, bucketName, key}, "no response object returned when checking if file exists"); - err = new Error(`no response from s3 ${bucketName} ${key}`); - return callback(err); - } - const exists = res.statusCode === 200; - logger.log({bucketName, key, exists}, "checked if file exsists in s3"); - return callback(err, exists); - }); - }, - - directorySize(bucketName, key, callback){ - logger.log({bucketName, key}, "get project size in s3"); - const s3Client = getKnoxClient(bucketName); - return s3Client.list({prefix:key}, function(err, data){ - if (err != null) { - logger.err({err, bucketName, key}, "something went wrong listing prefix in aws"); - return callback(err); - } - let totalSize = 0; - _.each(data.Contents, entry => totalSize += entry.Size); - logger.log({totalSize}, "total size"); - return callback(null, totalSize); - }); - } -}; + directorySize(bucketName, key, callback) { + logger.log({ bucketName, key }, 'get project size in s3') + const s3Client = getKnoxClient(bucketName) + return s3Client.list({ prefix: key }, function(err, data) { + if (err != null) { + logger.err( + { err, bucketName, key }, + 'something went wrong listing prefix in aws' + ) + return callback(err) + } + let totalSize = 0 + _.each(data.Contents, entry => (totalSize += entry.Size)) + logger.log({ totalSize }, 'total size') + return callback(null, totalSize) + }) + } +} diff --git a/services/filestore/app/js/SafeExec.js b/services/filestore/app/js/SafeExec.js index 4f13785460..dbc1576a88 100644 --- a/services/filestore/app/js/SafeExec.js +++ b/services/filestore/app/js/SafeExec.js @@ -12,10 +12,10 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const _ = require("underscore"); -const logger = require("logger-sharelatex"); -const child_process = require('child_process'); -const Settings = require("settings-sharelatex"); +const _ = require('underscore') +const logger = require('logger-sharelatex') +const child_process = require('child_process') +const Settings = require('settings-sharelatex') // execute a command in the same way as 'exec' but with a timeout that // kills all child processes @@ -24,44 +24,50 @@ const Settings = require("settings-sharelatex"); // group, then we can kill everything in that process group. module.exports = function(command, options, callback) { - if (callback == null) { callback = function(err, stdout, stderr) {}; } - if (!Settings.enableConversions) { - const error = new Error("Image conversions are disabled"); - return callback(error); - } + if (callback == null) { + callback = function(err, stdout, stderr) {} + } + if (!Settings.enableConversions) { + const error = new Error('Image conversions are disabled') + return callback(error) + } - // options are {timeout: number-of-milliseconds, killSignal: signal-name} - const [cmd, ...args] = Array.from(command); + // options are {timeout: number-of-milliseconds, killSignal: signal-name} + const [cmd, ...args] = Array.from(command) - const child = child_process.spawn(cmd, args, {detached:true}); - let stdout = ""; - let stderr = ""; + const child = child_process.spawn(cmd, args, { detached: true }) + let stdout = '' + let stderr = '' - const cleanup = _.once(function(err) { - if (killTimer != null) { clearTimeout(killTimer); } - return callback(err, stdout, stderr); - }); + const cleanup = _.once(function(err) { + if (killTimer != null) { + clearTimeout(killTimer) + } + return callback(err, stdout, stderr) + }) - if (options.timeout != null) { - var killTimer = setTimeout(function() { - try { - // use negative process id to kill process group - return process.kill(-child.pid, options.killSignal || "SIGTERM"); - } catch (error) { - return logger.log({process: child.pid, kill_error: error}, "error killing process"); - } - } - , options.timeout); - } + if (options.timeout != null) { + var killTimer = setTimeout(function() { + try { + // use negative process id to kill process group + return process.kill(-child.pid, options.killSignal || 'SIGTERM') + } catch (error) { + return logger.log( + { process: child.pid, kill_error: error }, + 'error killing process' + ) + } + }, options.timeout) + } - child.on('close', function(code, signal) { - const err = code ? new Error(`exit status ${code}`) : signal; - return cleanup(err); - }); + child.on('close', function(code, signal) { + const err = code ? new Error(`exit status ${code}`) : signal + return cleanup(err) + }) - child.on('error', err => cleanup(err)); + child.on('error', err => cleanup(err)) - child.stdout.on('data', chunk => stdout += chunk); + child.stdout.on('data', chunk => (stdout += chunk)) - return child.stderr.on('data', chunk => stderr += chunk); -}; + return child.stderr.on('data', chunk => (stderr += chunk)) +} From 1c7d1af4cae906efc6887c5e05759ed1dc9fdad5 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Mon, 16 Dec 2019 10:42:35 +0000 Subject: [PATCH 12/73] decaffeinate: Rename FilestoreApp.coffee and 1 other file from .coffee to .js --- .../acceptance/coffee/{FilestoreApp.coffee => FilestoreApp.js} | 0 .../coffee/{SendingFileTest.coffee => SendingFileTest.js} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename services/filestore/test/acceptance/coffee/{FilestoreApp.coffee => FilestoreApp.js} (100%) rename services/filestore/test/acceptance/coffee/{SendingFileTest.coffee => SendingFileTest.js} (100%) diff --git a/services/filestore/test/acceptance/coffee/FilestoreApp.coffee b/services/filestore/test/acceptance/coffee/FilestoreApp.js similarity index 100% rename from services/filestore/test/acceptance/coffee/FilestoreApp.coffee rename to services/filestore/test/acceptance/coffee/FilestoreApp.js diff --git a/services/filestore/test/acceptance/coffee/SendingFileTest.coffee b/services/filestore/test/acceptance/coffee/SendingFileTest.js similarity index 100% rename from services/filestore/test/acceptance/coffee/SendingFileTest.coffee rename to services/filestore/test/acceptance/coffee/SendingFileTest.js From e6e75d4e19f697b5274ba96f1e9d48e1752e24b2 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Mon, 16 Dec 2019 10:42:36 +0000 Subject: [PATCH 13/73] decaffeinate: Convert FilestoreApp.coffee and 1 other file to JS --- .../test/acceptance/coffee/FilestoreApp.js | 102 ++-- .../test/acceptance/coffee/SendingFileTest.js | 454 ++++++++++-------- 2 files changed, 332 insertions(+), 224 deletions(-) diff --git a/services/filestore/test/acceptance/coffee/FilestoreApp.js b/services/filestore/test/acceptance/coffee/FilestoreApp.js index 1b4cc38834..255c5a12ac 100644 --- a/services/filestore/test/acceptance/coffee/FilestoreApp.js +++ b/services/filestore/test/acceptance/coffee/FilestoreApp.js @@ -1,45 +1,73 @@ -app = require('../../../app') -require("logger-sharelatex").logger.level("info") -logger = require("logger-sharelatex") -Settings = require("settings-sharelatex") -request = require('request') +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS103: Rewrite code to no longer use __guard__ + * DS205: Consider reworking code to avoid use of IIFEs + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const app = require('../../../app'); +require("logger-sharelatex").logger.level("info"); +const logger = require("logger-sharelatex"); +const Settings = require("settings-sharelatex"); +const request = require('request'); -S3_TRIES = 30 +const S3_TRIES = 30; -module.exports = - running: false - initing: false - callbacks: [] - ensureRunning: (callback = (error) ->) -> - if @running - return callback() - else if @initing - @callbacks.push callback - else - @initing = true - @callbacks.push callback - app.listen Settings.internal?.filestore?.port, "localhost", (error) => - throw error if error? - @running = true - logger.log("filestore running in dev mode") +module.exports = { + running: false, + initing: false, + callbacks: [], + ensureRunning(callback) { + if (callback == null) { callback = function(error) {}; } + if (this.running) { + return callback(); + } else if (this.initing) { + return this.callbacks.push(callback); + } else { + this.initing = true; + this.callbacks.push(callback); + return app.listen(__guard__(Settings.internal != null ? Settings.internal.filestore : undefined, x => x.port), "localhost", error => { + if (error != null) { throw error; } + this.running = true; + logger.log("filestore running in dev mode"); - for callback in @callbacks - callback() + return (() => { + const result = []; + for (callback of Array.from(this.callbacks)) { + result.push(callback()); + } + return result; + })(); + }); + } + }, - waitForS3: (callback, tries) -> - return callback() unless Settings.filestore.s3?.endpoint - tries = 1 unless tries + waitForS3(callback, tries) { + if (!(Settings.filestore.s3 != null ? Settings.filestore.s3.endpoint : undefined)) { return callback(); } + if (!tries) { tries = 1; } - request.get "#{Settings.filestore.s3.endpoint}/", (err, response) => - console.log(err, response?.statusCode, tries) - if !err && [200, 404].includes(response?.statusCode) - return callback() + return request.get(`${Settings.filestore.s3.endpoint}/`, (err, response) => { + console.log(err, response != null ? response.statusCode : undefined, tries); + if (!err && [200, 404].includes(response != null ? response.statusCode : undefined)) { + return callback(); + } - if tries == S3_TRIES - return callback('timed out waiting for S3') + if (tries === S3_TRIES) { + return callback('timed out waiting for S3'); + } - setTimeout( - () => - @waitForS3 callback, tries + 1 + return setTimeout( + () => { + return this.waitForS3(callback, tries + 1); + }, 1000 - ) + ); + }); + } +}; + +function __guard__(value, transform) { + return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; +} \ No newline at end of file diff --git a/services/filestore/test/acceptance/coffee/SendingFileTest.js b/services/filestore/test/acceptance/coffee/SendingFileTest.js index 4e9443fd88..c45ef4d63d 100644 --- a/services/filestore/test/acceptance/coffee/SendingFileTest.js +++ b/services/filestore/test/acceptance/coffee/SendingFileTest.js @@ -1,226 +1,306 @@ -assert = require("chai").assert -sinon = require('sinon') -chai = require('chai') -should = chai.should() -expect = chai.expect -modulePath = "../../../app/js/LocalFileWriter.js" -SandboxedModule = require('sandboxed-module') -fs = require("fs") -request = require("request") -settings = require("settings-sharelatex") -FilestoreApp = require "./FilestoreApp" -async = require('async') +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS103: Rewrite code to no longer use __guard__ + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const { + assert +} = require("chai"); +const sinon = require('sinon'); +const chai = require('chai'); +const should = chai.should(); +const { + expect +} = chai; +const modulePath = "../../../app/js/LocalFileWriter.js"; +const SandboxedModule = require('sandboxed-module'); +const fs = require("fs"); +const request = require("request"); +const settings = require("settings-sharelatex"); +const FilestoreApp = require("./FilestoreApp"); +const async = require('async'); -getMetric = (filestoreUrl, metric, cb) -> - request.get "#{filestoreUrl}/metrics", (err, res) -> - expect(res.statusCode).to.equal 200 - metricRegex = new RegExp("^#{metric}{[^}]+} ([0-9]+)$", "m") - cb(parseInt(metricRegex.exec(res.body)?[1] || '0')) +const getMetric = (filestoreUrl, metric, cb) => request.get(`${filestoreUrl}/metrics`, function(err, res) { + expect(res.statusCode).to.equal(200); + const metricRegex = new RegExp(`^${metric}{[^}]+} ([0-9]+)$`, "m"); + return cb(parseInt(__guard__(metricRegex.exec(res.body), x => x[1]) || '0')); +}); -describe "Filestore", -> - before (done)-> - @localFileReadPath = "/tmp/filestore_acceptence_tests_file_read.txt" - @localFileWritePath = "/tmp/filestore_acceptence_tests_file_write.txt" +describe("Filestore", function() { + before(function(done){ + this.localFileReadPath = "/tmp/filestore_acceptence_tests_file_read.txt"; + this.localFileWritePath = "/tmp/filestore_acceptence_tests_file_write.txt"; - @constantFileContent = [ - "hello world" - "line 2 goes here #{Math.random()}" + this.constantFileContent = [ + "hello world", + `line 2 goes here ${Math.random()}`, "there are 3 lines in all" - ].join("\n") + ].join("\n"); - @filestoreUrl = "http://localhost:#{settings.internal.filestore.port}" - fs.writeFile @localFileReadPath, @constantFileContent, (err) -> - return done(err) if err - FilestoreApp.waitForS3(done) + this.filestoreUrl = `http://localhost:${settings.internal.filestore.port}`; + return fs.writeFile(this.localFileReadPath, this.constantFileContent, function(err) { + if (err) { return done(err); } + return FilestoreApp.waitForS3(done); + }); + }); - beforeEach (done)-> - FilestoreApp.ensureRunning => - async.parallel [ - (cb) => - fs.unlink @localFileWritePath, () -> - cb() - (cb) => - getMetric @filestoreUrl, 's3_egress', (metric) => - @previousEgress = metric - cb() - (cb) => - getMetric @filestoreUrl, 's3_ingress', (metric) => - @previousIngress = metric - cb() - ], done + beforeEach(function(done){ + return FilestoreApp.ensureRunning(() => { + return async.parallel([ + cb => { + return fs.unlink(this.localFileWritePath, () => cb()); + }, + cb => { + return getMetric(this.filestoreUrl, 's3_egress', metric => { + this.previousEgress = metric; + return cb(); + }); + }, + cb => { + return getMetric(this.filestoreUrl, 's3_ingress', metric => { + this.previousIngress = metric; + return cb(); + }); + } + ], done); + }); + }); - it "should send a 200 for status endpoint", (done)-> - request "#{@filestoreUrl}/status", (err, response, body)-> - response.statusCode.should.equal 200 - body.indexOf("filestore").should.not.equal -1 - body.indexOf("up").should.not.equal -1 - done() + it("should send a 200 for status endpoint", function(done){ + return request(`${this.filestoreUrl}/status`, function(err, response, body){ + response.statusCode.should.equal(200); + body.indexOf("filestore").should.not.equal(-1); + body.indexOf("up").should.not.equal(-1); + return done(); + }); + }); - describe "with a file on the server", -> + describe("with a file on the server", function() { - beforeEach (done)-> - @timeout(1000 * 10) - @file_id = Math.random() - @fileUrl = "#{@filestoreUrl}/project/acceptence_tests/file/#{@file_id}" + beforeEach(function(done){ + this.timeout(1000 * 10); + this.file_id = Math.random(); + this.fileUrl = `${this.filestoreUrl}/project/acceptence_tests/file/${this.file_id}`; - writeStream = request.post(@fileUrl) + const writeStream = request.post(this.fileUrl); - writeStream.on "end", done - fs.createReadStream(@localFileReadPath).pipe writeStream + writeStream.on("end", done); + return fs.createReadStream(this.localFileReadPath).pipe(writeStream); + }); - it "should return 404 for a non-existant id", (done) -> - @timeout(1000 * 20) - options = - uri: @fileUrl + '___this_is_clearly_wrong___' - request.get options, (err, response, body) => - response.statusCode.should.equal 404 - done() + it("should return 404 for a non-existant id", function(done) { + this.timeout(1000 * 20); + const options = + {uri: this.fileUrl + '___this_is_clearly_wrong___'}; + return request.get(options, (err, response, body) => { + response.statusCode.should.equal(404); + return done(); + }); + }); - it 'should record an egress metric for the upload', (done) -> - getMetric @filestoreUrl, 's3_egress', (metric) => - expect(metric - @previousEgress).to.equal @constantFileContent.length - done() + it('should record an egress metric for the upload', function(done) { + return getMetric(this.filestoreUrl, 's3_egress', metric => { + expect(metric - this.previousEgress).to.equal(this.constantFileContent.length); + return done(); + }); + }); - it "should return the file size on a HEAD request", (done) -> - expectedLength = Buffer.byteLength(@constantFileContent) - request.head @fileUrl, (err, res) => - expect(res.statusCode).to.equal(200) - expect(res.headers['content-length']).to.equal(expectedLength.toString()) - done() + it("should return the file size on a HEAD request", function(done) { + const expectedLength = Buffer.byteLength(this.constantFileContent); + return request.head(this.fileUrl, (err, res) => { + expect(res.statusCode).to.equal(200); + expect(res.headers['content-length']).to.equal(expectedLength.toString()); + return done(); + }); + }); - it "should be able get the file back", (done)-> - @timeout(1000 * 10) - request.get @fileUrl, (err, response, body)=> - body.should.equal @constantFileContent - done() + it("should be able get the file back", function(done){ + this.timeout(1000 * 10); + return request.get(this.fileUrl, (err, response, body)=> { + body.should.equal(this.constantFileContent); + return done(); + }); + }); - it "should record an ingress metric when downloading the file", (done)-> - @timeout(1000 * 10) - request.get @fileUrl, () => - getMetric @filestoreUrl, 's3_ingress', (metric) => - expect(metric - @previousIngress).to.equal @constantFileContent.length - done() + it("should record an ingress metric when downloading the file", function(done){ + this.timeout(1000 * 10); + return request.get(this.fileUrl, () => { + return getMetric(this.filestoreUrl, 's3_ingress', metric => { + expect(metric - this.previousIngress).to.equal(this.constantFileContent.length); + return done(); + }); + }); + }); - it "should be able to get back the first 9 bytes of the file", (done) -> - @timeout(1000 * 10) - options = - uri: @fileUrl - headers: + it("should be able to get back the first 9 bytes of the file", function(done) { + this.timeout(1000 * 10); + const options = { + uri: this.fileUrl, + headers: { 'Range': 'bytes=0-8' - request.get options, (err, response, body)=> - body.should.equal 'hello wor' - done() + } + }; + return request.get(options, (err, response, body)=> { + body.should.equal('hello wor'); + return done(); + }); + }); - it "should record an ingress metric for a partial download", (done)-> - @timeout(1000 * 10) - options = - uri: @fileUrl - headers: + it("should record an ingress metric for a partial download", function(done){ + this.timeout(1000 * 10); + const options = { + uri: this.fileUrl, + headers: { 'Range': 'bytes=0-8' - request.get options, ()=> - getMetric @filestoreUrl, 's3_ingress', (metric) => - expect(metric - @previousIngress).to.equal 9 - done() + } + }; + return request.get(options, ()=> { + return getMetric(this.filestoreUrl, 's3_ingress', metric => { + expect(metric - this.previousIngress).to.equal(9); + return done(); + }); + }); + }); - it "should be able to get back bytes 4 through 10 of the file", (done) -> - @timeout(1000 * 10) - options = - uri: @fileUrl - headers: + it("should be able to get back bytes 4 through 10 of the file", function(done) { + this.timeout(1000 * 10); + const options = { + uri: this.fileUrl, + headers: { 'Range': 'bytes=4-10' - request.get options, (err, response, body)=> - body.should.equal 'o world' - done() + } + }; + return request.get(options, (err, response, body)=> { + body.should.equal('o world'); + return done(); + }); + }); - it "should be able to delete the file", (done)-> - @timeout(1000 * 20) - request.del @fileUrl, (err, response, body)=> - response.statusCode.should.equal 204 - request.get @fileUrl, (err, response, body)=> - response.statusCode.should.equal 404 - done() + it("should be able to delete the file", function(done){ + this.timeout(1000 * 20); + return request.del(this.fileUrl, (err, response, body)=> { + response.statusCode.should.equal(204); + return request.get(this.fileUrl, (err, response, body)=> { + response.statusCode.should.equal(404); + return done(); + }); + }); + }); - it "should be able to copy files", (done)-> - @timeout(1000 * 20) + return it("should be able to copy files", function(done){ + this.timeout(1000 * 20); - newProjectID = "acceptence_tests_copyied_project" - newFileId = Math.random() - newFileUrl = "#{@filestoreUrl}/project/#{newProjectID}/file/#{newFileId}" - opts = - method: 'put' - uri: newFileUrl - json: - source: - project_id:"acceptence_tests" - file_id: @file_id - request opts, (err, response, body)=> - response.statusCode.should.equal 200 - request.del @fileUrl, (err, response, body)=> - response.statusCode.should.equal 204 - request.get newFileUrl, (err, response, body)=> - body.should.equal @constantFileContent - done() + const newProjectID = "acceptence_tests_copyied_project"; + const newFileId = Math.random(); + const newFileUrl = `${this.filestoreUrl}/project/${newProjectID}/file/${newFileId}`; + const opts = { + method: 'put', + uri: newFileUrl, + json: { + source: { + project_id:"acceptence_tests", + file_id: this.file_id + } + } + }; + return request(opts, (err, response, body)=> { + response.statusCode.should.equal(200); + return request.del(this.fileUrl, (err, response, body)=> { + response.statusCode.should.equal(204); + return request.get(newFileUrl, (err, response, body)=> { + body.should.equal(this.constantFileContent); + return done(); + }); + }); + }); + }); + }); - describe "with a pdf file", -> + return describe("with a pdf file", function() { - beforeEach (done)-> - @timeout(1000 * 10) - @file_id = Math.random() - @fileUrl = "#{@filestoreUrl}/project/acceptence_tests/file/#{@file_id}" - @localFileReadPath = __dirname + '/../../fixtures/test.pdf' - fs.stat @localFileReadPath, (err, stat) => - @localFileSize = stat.size - writeStream = request.post(@fileUrl) + beforeEach(function(done){ + this.timeout(1000 * 10); + this.file_id = Math.random(); + this.fileUrl = `${this.filestoreUrl}/project/acceptence_tests/file/${this.file_id}`; + this.localFileReadPath = __dirname + '/../../fixtures/test.pdf'; + return fs.stat(this.localFileReadPath, (err, stat) => { + this.localFileSize = stat.size; + const writeStream = request.post(this.fileUrl); - writeStream.on "end", done - fs.createReadStream(@localFileReadPath).pipe writeStream + writeStream.on("end", done); + return fs.createReadStream(this.localFileReadPath).pipe(writeStream); + }); + }); - it 'should record an egress metric for the upload', (done) -> - getMetric @filestoreUrl, 's3_egress', (metric) => - expect(metric - @previousEgress).to.equal @localFileSize - done() + it('should record an egress metric for the upload', function(done) { + return getMetric(this.filestoreUrl, 's3_egress', metric => { + expect(metric - this.previousEgress).to.equal(this.localFileSize); + return done(); + }); + }); - it "should be able get the file back", (done)-> - @timeout(1000 * 10) - request.get @fileUrl, (err, response, body)=> - expect(body.substring(0, 8)).to.equal '%PDF-1.5' - done() + it("should be able get the file back", function(done){ + this.timeout(1000 * 10); + return request.get(this.fileUrl, (err, response, body)=> { + expect(body.substring(0, 8)).to.equal('%PDF-1.5'); + return done(); + }); + }); - describe "getting the preview image", -> + describe("getting the preview image", function() { - beforeEach -> - @previewFileUrl = "#{@fileUrl}?style=preview" + beforeEach(function() { + return this.previewFileUrl = `${this.fileUrl}?style=preview`; + }); - it "should not time out", (done) -> - @timeout(1000 * 20) - request.get @previewFileUrl, (err, response, body) => - expect(response).to.not.equal null - done() + it("should not time out", function(done) { + this.timeout(1000 * 20); + return request.get(this.previewFileUrl, (err, response, body) => { + expect(response).to.not.equal(null); + return done(); + }); + }); - it "should respond with image data", (done) -> - # note: this test relies of the imagemagick conversion working - @timeout(1000 * 20) - request.get @previewFileUrl, (err, response, body) => - expect(response.statusCode).to.equal 200 - expect(body.length).to.be.greaterThan 400 - done() + return it("should respond with image data", function(done) { + // note: this test relies of the imagemagick conversion working + this.timeout(1000 * 20); + return request.get(this.previewFileUrl, (err, response, body) => { + expect(response.statusCode).to.equal(200); + expect(body.length).to.be.greaterThan(400); + return done(); + }); + }); + }); - describe "warming the cache", -> + return describe("warming the cache", function() { - beforeEach -> - @fileUrl = @fileUrl + '?style=preview&cacheWarm=true' + beforeEach(function() { + return this.fileUrl = this.fileUrl + '?style=preview&cacheWarm=true'; + }); - it "should not time out", (done) -> - @timeout(1000 * 20) - request.get @fileUrl, (err, response, body) => - expect(response).to.not.equal null - done() + it("should not time out", function(done) { + this.timeout(1000 * 20); + return request.get(this.fileUrl, (err, response, body) => { + expect(response).to.not.equal(null); + return done(); + }); + }); - it "should respond with only an 'OK'", (done) -> - # note: this test relies of the imagemagick conversion working - @timeout(1000 * 20) - request.get @fileUrl, (err, response, body) => - expect(response.statusCode).to.equal 200 - body.should.equal 'OK' - done() + return it("should respond with only an 'OK'", function(done) { + // note: this test relies of the imagemagick conversion working + this.timeout(1000 * 20); + return request.get(this.fileUrl, (err, response, body) => { + expect(response.statusCode).to.equal(200); + body.should.equal('OK'); + return done(); + }); + }); + }); + }); +}); + +function __guard__(value, transform) { + return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; +} \ No newline at end of file From 738e1d821e491250a7e3f8c1ab4f9da8060e5356 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Mon, 16 Dec 2019 10:42:36 +0000 Subject: [PATCH 14/73] decaffeinate: Run post-processing cleanups on FilestoreApp.coffee and 1 other file --- services/filestore/test/acceptance/coffee/FilestoreApp.js | 6 ++++++ .../filestore/test/acceptance/coffee/SendingFileTest.js | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/services/filestore/test/acceptance/coffee/FilestoreApp.js b/services/filestore/test/acceptance/coffee/FilestoreApp.js index 255c5a12ac..467d66ee6e 100644 --- a/services/filestore/test/acceptance/coffee/FilestoreApp.js +++ b/services/filestore/test/acceptance/coffee/FilestoreApp.js @@ -1,3 +1,9 @@ +/* eslint-disable + handle-callback-err, + standard/no-callback-literal, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS101: Remove unnecessary use of Array.from diff --git a/services/filestore/test/acceptance/coffee/SendingFileTest.js b/services/filestore/test/acceptance/coffee/SendingFileTest.js index c45ef4d63d..2f31e60b99 100644 --- a/services/filestore/test/acceptance/coffee/SendingFileTest.js +++ b/services/filestore/test/acceptance/coffee/SendingFileTest.js @@ -1,3 +1,11 @@ +/* eslint-disable + handle-callback-err, + no-path-concat, + no-return-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns From b8e7abd25eba052017d336f1a7f13b5b3516da11 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 16 Dec 2019 10:42:40 +0000 Subject: [PATCH 15/73] Rename test/acceptance/coffee to test/acceptance/js --- services/filestore/test/acceptance/{coffee => js}/FilestoreApp.js | 0 .../filestore/test/acceptance/{coffee => js}/SendingFileTest.js | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename services/filestore/test/acceptance/{coffee => js}/FilestoreApp.js (100%) rename services/filestore/test/acceptance/{coffee => js}/SendingFileTest.js (100%) diff --git a/services/filestore/test/acceptance/coffee/FilestoreApp.js b/services/filestore/test/acceptance/js/FilestoreApp.js similarity index 100% rename from services/filestore/test/acceptance/coffee/FilestoreApp.js rename to services/filestore/test/acceptance/js/FilestoreApp.js diff --git a/services/filestore/test/acceptance/coffee/SendingFileTest.js b/services/filestore/test/acceptance/js/SendingFileTest.js similarity index 100% rename from services/filestore/test/acceptance/coffee/SendingFileTest.js rename to services/filestore/test/acceptance/js/SendingFileTest.js From e4b110676166dd5898d58231af7860d6ebbf45c7 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 16 Dec 2019 10:42:41 +0000 Subject: [PATCH 16/73] Prettier: convert test/acceptance decaffeinated files to Prettier format --- .../test/acceptance/js/FilestoreApp.js | 138 +++-- .../test/acceptance/js/SendingFileTest.js | 548 +++++++++--------- 2 files changed, 364 insertions(+), 322 deletions(-) diff --git a/services/filestore/test/acceptance/js/FilestoreApp.js b/services/filestore/test/acceptance/js/FilestoreApp.js index 467d66ee6e..3a5103d5f6 100644 --- a/services/filestore/test/acceptance/js/FilestoreApp.js +++ b/services/filestore/test/acceptance/js/FilestoreApp.js @@ -13,67 +13,97 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const app = require('../../../app'); -require("logger-sharelatex").logger.level("info"); -const logger = require("logger-sharelatex"); -const Settings = require("settings-sharelatex"); -const request = require('request'); +const app = require('../../../app') +require('logger-sharelatex').logger.level('info') +const logger = require('logger-sharelatex') +const Settings = require('settings-sharelatex') +const request = require('request') -const S3_TRIES = 30; +const S3_TRIES = 30 module.exports = { - running: false, - initing: false, - callbacks: [], - ensureRunning(callback) { - if (callback == null) { callback = function(error) {}; } - if (this.running) { - return callback(); - } else if (this.initing) { - return this.callbacks.push(callback); - } else { - this.initing = true; - this.callbacks.push(callback); - return app.listen(__guard__(Settings.internal != null ? Settings.internal.filestore : undefined, x => x.port), "localhost", error => { - if (error != null) { throw error; } - this.running = true; - logger.log("filestore running in dev mode"); + running: false, + initing: false, + callbacks: [], + ensureRunning(callback) { + if (callback == null) { + callback = function(error) {} + } + if (this.running) { + return callback() + } else if (this.initing) { + return this.callbacks.push(callback) + } else { + this.initing = true + this.callbacks.push(callback) + return app.listen( + __guard__( + Settings.internal != null ? Settings.internal.filestore : undefined, + x => x.port + ), + 'localhost', + error => { + if (error != null) { + throw error + } + this.running = true + logger.log('filestore running in dev mode') - return (() => { - const result = []; - for (callback of Array.from(this.callbacks)) { - result.push(callback()); - } - return result; - })(); - }); - } - }, + return (() => { + const result = [] + for (callback of Array.from(this.callbacks)) { + result.push(callback()) + } + return result + })() + } + ) + } + }, - waitForS3(callback, tries) { - if (!(Settings.filestore.s3 != null ? Settings.filestore.s3.endpoint : undefined)) { return callback(); } - if (!tries) { tries = 1; } + waitForS3(callback, tries) { + if ( + !(Settings.filestore.s3 != null + ? Settings.filestore.s3.endpoint + : undefined) + ) { + return callback() + } + if (!tries) { + tries = 1 + } - return request.get(`${Settings.filestore.s3.endpoint}/`, (err, response) => { - console.log(err, response != null ? response.statusCode : undefined, tries); - if (!err && [200, 404].includes(response != null ? response.statusCode : undefined)) { - return callback(); - } + return request.get( + `${Settings.filestore.s3.endpoint}/`, + (err, response) => { + console.log( + err, + response != null ? response.statusCode : undefined, + tries + ) + if ( + !err && + [200, 404].includes( + response != null ? response.statusCode : undefined + ) + ) { + return callback() + } - if (tries === S3_TRIES) { - return callback('timed out waiting for S3'); - } + if (tries === S3_TRIES) { + return callback('timed out waiting for S3') + } - return setTimeout( - () => { - return this.waitForS3(callback, tries + 1); - }, - 1000 - ); - }); - } -}; + return setTimeout(() => { + return this.waitForS3(callback, tries + 1) + }, 1000) + } + ) + } +} function __guard__(value, transform) { - return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; -} \ No newline at end of file + return typeof value !== 'undefined' && value !== null + ? transform(value) + : undefined +} diff --git a/services/filestore/test/acceptance/js/SendingFileTest.js b/services/filestore/test/acceptance/js/SendingFileTest.js index 2f31e60b99..c20fa01c42 100644 --- a/services/filestore/test/acceptance/js/SendingFileTest.js +++ b/services/filestore/test/acceptance/js/SendingFileTest.js @@ -12,303 +12,315 @@ * DS103: Rewrite code to no longer use __guard__ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const { - assert -} = require("chai"); -const sinon = require('sinon'); -const chai = require('chai'); -const should = chai.should(); -const { - expect -} = chai; -const modulePath = "../../../app/js/LocalFileWriter.js"; -const SandboxedModule = require('sandboxed-module'); -const fs = require("fs"); -const request = require("request"); -const settings = require("settings-sharelatex"); -const FilestoreApp = require("./FilestoreApp"); -const async = require('async'); +const { assert } = require('chai') +const sinon = require('sinon') +const chai = require('chai') +const should = chai.should() +const { expect } = chai +const modulePath = '../../../app/js/LocalFileWriter.js' +const SandboxedModule = require('sandboxed-module') +const fs = require('fs') +const request = require('request') +const settings = require('settings-sharelatex') +const FilestoreApp = require('./FilestoreApp') +const async = require('async') +const getMetric = (filestoreUrl, metric, cb) => + request.get(`${filestoreUrl}/metrics`, function(err, res) { + expect(res.statusCode).to.equal(200) + const metricRegex = new RegExp(`^${metric}{[^}]+} ([0-9]+)$`, 'm') + return cb(parseInt(__guard__(metricRegex.exec(res.body), x => x[1]) || '0')) + }) -const getMetric = (filestoreUrl, metric, cb) => request.get(`${filestoreUrl}/metrics`, function(err, res) { - expect(res.statusCode).to.equal(200); - const metricRegex = new RegExp(`^${metric}{[^}]+} ([0-9]+)$`, "m"); - return cb(parseInt(__guard__(metricRegex.exec(res.body), x => x[1]) || '0')); -}); +describe('Filestore', function() { + before(function(done) { + this.localFileReadPath = '/tmp/filestore_acceptence_tests_file_read.txt' + this.localFileWritePath = '/tmp/filestore_acceptence_tests_file_write.txt' -describe("Filestore", function() { - before(function(done){ - this.localFileReadPath = "/tmp/filestore_acceptence_tests_file_read.txt"; - this.localFileWritePath = "/tmp/filestore_acceptence_tests_file_write.txt"; + this.constantFileContent = [ + 'hello world', + `line 2 goes here ${Math.random()}`, + 'there are 3 lines in all' + ].join('\n') - this.constantFileContent = [ - "hello world", - `line 2 goes here ${Math.random()}`, - "there are 3 lines in all" - ].join("\n"); + this.filestoreUrl = `http://localhost:${settings.internal.filestore.port}` + return fs.writeFile( + this.localFileReadPath, + this.constantFileContent, + function(err) { + if (err) { + return done(err) + } + return FilestoreApp.waitForS3(done) + } + ) + }) - this.filestoreUrl = `http://localhost:${settings.internal.filestore.port}`; - return fs.writeFile(this.localFileReadPath, this.constantFileContent, function(err) { - if (err) { return done(err); } - return FilestoreApp.waitForS3(done); - }); - }); + beforeEach(function(done) { + return FilestoreApp.ensureRunning(() => { + return async.parallel( + [ + cb => { + return fs.unlink(this.localFileWritePath, () => cb()) + }, + cb => { + return getMetric(this.filestoreUrl, 's3_egress', metric => { + this.previousEgress = metric + return cb() + }) + }, + cb => { + return getMetric(this.filestoreUrl, 's3_ingress', metric => { + this.previousIngress = metric + return cb() + }) + } + ], + done + ) + }) + }) - beforeEach(function(done){ - return FilestoreApp.ensureRunning(() => { - return async.parallel([ - cb => { - return fs.unlink(this.localFileWritePath, () => cb()); - }, - cb => { - return getMetric(this.filestoreUrl, 's3_egress', metric => { - this.previousEgress = metric; - return cb(); - }); - }, - cb => { - return getMetric(this.filestoreUrl, 's3_ingress', metric => { - this.previousIngress = metric; - return cb(); - }); - } - ], done); - }); - }); + it('should send a 200 for status endpoint', function(done) { + return request(`${this.filestoreUrl}/status`, function( + err, + response, + body + ) { + response.statusCode.should.equal(200) + body.indexOf('filestore').should.not.equal(-1) + body.indexOf('up').should.not.equal(-1) + return done() + }) + }) - it("should send a 200 for status endpoint", function(done){ - return request(`${this.filestoreUrl}/status`, function(err, response, body){ - response.statusCode.should.equal(200); - body.indexOf("filestore").should.not.equal(-1); - body.indexOf("up").should.not.equal(-1); - return done(); - }); - }); + describe('with a file on the server', function() { + beforeEach(function(done) { + this.timeout(1000 * 10) + this.file_id = Math.random() + this.fileUrl = `${this.filestoreUrl}/project/acceptence_tests/file/${this.file_id}` - describe("with a file on the server", function() { + const writeStream = request.post(this.fileUrl) - beforeEach(function(done){ - this.timeout(1000 * 10); - this.file_id = Math.random(); - this.fileUrl = `${this.filestoreUrl}/project/acceptence_tests/file/${this.file_id}`; + writeStream.on('end', done) + return fs.createReadStream(this.localFileReadPath).pipe(writeStream) + }) - const writeStream = request.post(this.fileUrl); + it('should return 404 for a non-existant id', function(done) { + this.timeout(1000 * 20) + const options = { uri: this.fileUrl + '___this_is_clearly_wrong___' } + return request.get(options, (err, response, body) => { + response.statusCode.should.equal(404) + return done() + }) + }) - writeStream.on("end", done); - return fs.createReadStream(this.localFileReadPath).pipe(writeStream); - }); + it('should record an egress metric for the upload', function(done) { + return getMetric(this.filestoreUrl, 's3_egress', metric => { + expect(metric - this.previousEgress).to.equal( + this.constantFileContent.length + ) + return done() + }) + }) - it("should return 404 for a non-existant id", function(done) { - this.timeout(1000 * 20); - const options = - {uri: this.fileUrl + '___this_is_clearly_wrong___'}; - return request.get(options, (err, response, body) => { - response.statusCode.should.equal(404); - return done(); - }); - }); + it('should return the file size on a HEAD request', function(done) { + const expectedLength = Buffer.byteLength(this.constantFileContent) + return request.head(this.fileUrl, (err, res) => { + expect(res.statusCode).to.equal(200) + expect(res.headers['content-length']).to.equal( + expectedLength.toString() + ) + return done() + }) + }) - it('should record an egress metric for the upload', function(done) { - return getMetric(this.filestoreUrl, 's3_egress', metric => { - expect(metric - this.previousEgress).to.equal(this.constantFileContent.length); - return done(); - }); - }); + it('should be able get the file back', function(done) { + this.timeout(1000 * 10) + return request.get(this.fileUrl, (err, response, body) => { + body.should.equal(this.constantFileContent) + return done() + }) + }) - it("should return the file size on a HEAD request", function(done) { - const expectedLength = Buffer.byteLength(this.constantFileContent); - return request.head(this.fileUrl, (err, res) => { - expect(res.statusCode).to.equal(200); - expect(res.headers['content-length']).to.equal(expectedLength.toString()); - return done(); - }); - }); + it('should record an ingress metric when downloading the file', function(done) { + this.timeout(1000 * 10) + return request.get(this.fileUrl, () => { + return getMetric(this.filestoreUrl, 's3_ingress', metric => { + expect(metric - this.previousIngress).to.equal( + this.constantFileContent.length + ) + return done() + }) + }) + }) - it("should be able get the file back", function(done){ - this.timeout(1000 * 10); - return request.get(this.fileUrl, (err, response, body)=> { - body.should.equal(this.constantFileContent); - return done(); - }); - }); + it('should be able to get back the first 9 bytes of the file', function(done) { + this.timeout(1000 * 10) + const options = { + uri: this.fileUrl, + headers: { + Range: 'bytes=0-8' + } + } + return request.get(options, (err, response, body) => { + body.should.equal('hello wor') + return done() + }) + }) - it("should record an ingress metric when downloading the file", function(done){ - this.timeout(1000 * 10); - return request.get(this.fileUrl, () => { - return getMetric(this.filestoreUrl, 's3_ingress', metric => { - expect(metric - this.previousIngress).to.equal(this.constantFileContent.length); - return done(); - }); - }); - }); + it('should record an ingress metric for a partial download', function(done) { + this.timeout(1000 * 10) + const options = { + uri: this.fileUrl, + headers: { + Range: 'bytes=0-8' + } + } + return request.get(options, () => { + return getMetric(this.filestoreUrl, 's3_ingress', metric => { + expect(metric - this.previousIngress).to.equal(9) + return done() + }) + }) + }) - it("should be able to get back the first 9 bytes of the file", function(done) { - this.timeout(1000 * 10); - const options = { - uri: this.fileUrl, - headers: { - 'Range': 'bytes=0-8' - } - }; - return request.get(options, (err, response, body)=> { - body.should.equal('hello wor'); - return done(); - }); - }); + it('should be able to get back bytes 4 through 10 of the file', function(done) { + this.timeout(1000 * 10) + const options = { + uri: this.fileUrl, + headers: { + Range: 'bytes=4-10' + } + } + return request.get(options, (err, response, body) => { + body.should.equal('o world') + return done() + }) + }) - it("should record an ingress metric for a partial download", function(done){ - this.timeout(1000 * 10); - const options = { - uri: this.fileUrl, - headers: { - 'Range': 'bytes=0-8' - } - }; - return request.get(options, ()=> { - return getMetric(this.filestoreUrl, 's3_ingress', metric => { - expect(metric - this.previousIngress).to.equal(9); - return done(); - }); - }); - }); + it('should be able to delete the file', function(done) { + this.timeout(1000 * 20) + return request.del(this.fileUrl, (err, response, body) => { + response.statusCode.should.equal(204) + return request.get(this.fileUrl, (err, response, body) => { + response.statusCode.should.equal(404) + return done() + }) + }) + }) - it("should be able to get back bytes 4 through 10 of the file", function(done) { - this.timeout(1000 * 10); - const options = { - uri: this.fileUrl, - headers: { - 'Range': 'bytes=4-10' - } - }; - return request.get(options, (err, response, body)=> { - body.should.equal('o world'); - return done(); - }); - }); + return it('should be able to copy files', function(done) { + this.timeout(1000 * 20) - it("should be able to delete the file", function(done){ - this.timeout(1000 * 20); - return request.del(this.fileUrl, (err, response, body)=> { - response.statusCode.should.equal(204); - return request.get(this.fileUrl, (err, response, body)=> { - response.statusCode.should.equal(404); - return done(); - }); - }); - }); + const newProjectID = 'acceptence_tests_copyied_project' + const newFileId = Math.random() + const newFileUrl = `${this.filestoreUrl}/project/${newProjectID}/file/${newFileId}` + const opts = { + method: 'put', + uri: newFileUrl, + json: { + source: { + project_id: 'acceptence_tests', + file_id: this.file_id + } + } + } + return request(opts, (err, response, body) => { + response.statusCode.should.equal(200) + return request.del(this.fileUrl, (err, response, body) => { + response.statusCode.should.equal(204) + return request.get(newFileUrl, (err, response, body) => { + body.should.equal(this.constantFileContent) + return done() + }) + }) + }) + }) + }) - return it("should be able to copy files", function(done){ - this.timeout(1000 * 20); + return describe('with a pdf file', function() { + beforeEach(function(done) { + this.timeout(1000 * 10) + this.file_id = Math.random() + this.fileUrl = `${this.filestoreUrl}/project/acceptence_tests/file/${this.file_id}` + this.localFileReadPath = __dirname + '/../../fixtures/test.pdf' + return fs.stat(this.localFileReadPath, (err, stat) => { + this.localFileSize = stat.size + const writeStream = request.post(this.fileUrl) - const newProjectID = "acceptence_tests_copyied_project"; - const newFileId = Math.random(); - const newFileUrl = `${this.filestoreUrl}/project/${newProjectID}/file/${newFileId}`; - const opts = { - method: 'put', - uri: newFileUrl, - json: { - source: { - project_id:"acceptence_tests", - file_id: this.file_id - } - } - }; - return request(opts, (err, response, body)=> { - response.statusCode.should.equal(200); - return request.del(this.fileUrl, (err, response, body)=> { - response.statusCode.should.equal(204); - return request.get(newFileUrl, (err, response, body)=> { - body.should.equal(this.constantFileContent); - return done(); - }); - }); - }); - }); - }); + writeStream.on('end', done) + return fs.createReadStream(this.localFileReadPath).pipe(writeStream) + }) + }) - return describe("with a pdf file", function() { + it('should record an egress metric for the upload', function(done) { + return getMetric(this.filestoreUrl, 's3_egress', metric => { + expect(metric - this.previousEgress).to.equal(this.localFileSize) + return done() + }) + }) - beforeEach(function(done){ - this.timeout(1000 * 10); - this.file_id = Math.random(); - this.fileUrl = `${this.filestoreUrl}/project/acceptence_tests/file/${this.file_id}`; - this.localFileReadPath = __dirname + '/../../fixtures/test.pdf'; - return fs.stat(this.localFileReadPath, (err, stat) => { - this.localFileSize = stat.size; - const writeStream = request.post(this.fileUrl); + it('should be able get the file back', function(done) { + this.timeout(1000 * 10) + return request.get(this.fileUrl, (err, response, body) => { + expect(body.substring(0, 8)).to.equal('%PDF-1.5') + return done() + }) + }) - writeStream.on("end", done); - return fs.createReadStream(this.localFileReadPath).pipe(writeStream); - }); - }); + describe('getting the preview image', function() { + beforeEach(function() { + return (this.previewFileUrl = `${this.fileUrl}?style=preview`) + }) - it('should record an egress metric for the upload', function(done) { - return getMetric(this.filestoreUrl, 's3_egress', metric => { - expect(metric - this.previousEgress).to.equal(this.localFileSize); - return done(); - }); - }); + it('should not time out', function(done) { + this.timeout(1000 * 20) + return request.get(this.previewFileUrl, (err, response, body) => { + expect(response).to.not.equal(null) + return done() + }) + }) - it("should be able get the file back", function(done){ - this.timeout(1000 * 10); - return request.get(this.fileUrl, (err, response, body)=> { - expect(body.substring(0, 8)).to.equal('%PDF-1.5'); - return done(); - }); - }); + return it('should respond with image data', function(done) { + // note: this test relies of the imagemagick conversion working + this.timeout(1000 * 20) + return request.get(this.previewFileUrl, (err, response, body) => { + expect(response.statusCode).to.equal(200) + expect(body.length).to.be.greaterThan(400) + return done() + }) + }) + }) - describe("getting the preview image", function() { + return describe('warming the cache', function() { + beforeEach(function() { + return (this.fileUrl = this.fileUrl + '?style=preview&cacheWarm=true') + }) - beforeEach(function() { - return this.previewFileUrl = `${this.fileUrl}?style=preview`; - }); + it('should not time out', function(done) { + this.timeout(1000 * 20) + return request.get(this.fileUrl, (err, response, body) => { + expect(response).to.not.equal(null) + return done() + }) + }) - it("should not time out", function(done) { - this.timeout(1000 * 20); - return request.get(this.previewFileUrl, (err, response, body) => { - expect(response).to.not.equal(null); - return done(); - }); - }); - - return it("should respond with image data", function(done) { - // note: this test relies of the imagemagick conversion working - this.timeout(1000 * 20); - return request.get(this.previewFileUrl, (err, response, body) => { - expect(response.statusCode).to.equal(200); - expect(body.length).to.be.greaterThan(400); - return done(); - }); - }); - }); - - return describe("warming the cache", function() { - - beforeEach(function() { - return this.fileUrl = this.fileUrl + '?style=preview&cacheWarm=true'; - }); - - it("should not time out", function(done) { - this.timeout(1000 * 20); - return request.get(this.fileUrl, (err, response, body) => { - expect(response).to.not.equal(null); - return done(); - }); - }); - - return it("should respond with only an 'OK'", function(done) { - // note: this test relies of the imagemagick conversion working - this.timeout(1000 * 20); - return request.get(this.fileUrl, (err, response, body) => { - expect(response.statusCode).to.equal(200); - body.should.equal('OK'); - return done(); - }); - }); - }); - }); -}); + return it("should respond with only an 'OK'", function(done) { + // note: this test relies of the imagemagick conversion working + this.timeout(1000 * 20) + return request.get(this.fileUrl, (err, response, body) => { + expect(response.statusCode).to.equal(200) + body.should.equal('OK') + return done() + }) + }) + }) + }) +}) function __guard__(value, transform) { - return (typeof value !== 'undefined' && value !== null) ? transform(value) : undefined; -} \ No newline at end of file + return typeof value !== 'undefined' && value !== null + ? transform(value) + : undefined +} From d70a33c4f57abdd1dc4b647b5cebded3a1cb23a8 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 16 Dec 2019 10:52:34 +0000 Subject: [PATCH 17/73] Rename individual coffee files to js files --- services/filestore/{app.coffee => app.js} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename services/filestore/{app.coffee => app.js} (100%) diff --git a/services/filestore/app.coffee b/services/filestore/app.js similarity index 100% rename from services/filestore/app.coffee rename to services/filestore/app.js From 1d295ef7e4ed74a88508f01355c68617ec479774 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 16 Dec 2019 10:52:40 +0000 Subject: [PATCH 18/73] Decaffeinate: convert individual files to js --- services/filestore/app.js | 254 +++++++++++++++++++++----------------- 1 file changed, 140 insertions(+), 114 deletions(-) diff --git a/services/filestore/app.js b/services/filestore/app.js index 6c21686826..8a5bd9a04a 100644 --- a/services/filestore/app.js +++ b/services/filestore/app.js @@ -1,147 +1,173 @@ -Metrics = require "metrics-sharelatex" -Metrics.initialize("filestore") -express = require('express') -bodyParser = require "body-parser" -logger = require('logger-sharelatex') -logger.initialize("filestore") -settings = require("settings-sharelatex") -request = require("request") -fileController = require("./app/js/FileController") -bucketController = require("./app/js/BucketController") -keyBuilder = require("./app/js/KeyBuilder") -healthCheckController = require("./app/js/HealthCheckController") -domain = require("domain") -appIsOk = true -app = express() +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const Metrics = require("metrics-sharelatex"); +Metrics.initialize("filestore"); +const express = require('express'); +const bodyParser = require("body-parser"); +let logger = require('logger-sharelatex'); +logger.initialize("filestore"); +const settings = require("settings-sharelatex"); +const request = require("request"); +const fileController = require("./app/js/FileController"); +const bucketController = require("./app/js/BucketController"); +const keyBuilder = require("./app/js/KeyBuilder"); +const healthCheckController = require("./app/js/HealthCheckController"); +const domain = require("domain"); +let appIsOk = true; +const app = express(); -if settings.sentry?.dsn? - logger.initializeErrorReporting(settings.sentry.dsn) +if ((settings.sentry != null ? settings.sentry.dsn : undefined) != null) { + logger.initializeErrorReporting(settings.sentry.dsn); +} -Metrics.open_sockets.monitor(logger) -Metrics.event_loop?.monitor(logger) -Metrics.memory.monitor(logger) +Metrics.open_sockets.monitor(logger); +if (Metrics.event_loop != null) { + Metrics.event_loop.monitor(logger); +} +Metrics.memory.monitor(logger); -app.use Metrics.http.monitor(logger) +app.use(Metrics.http.monitor(logger)); -app.use (req, res, next)-> - Metrics.inc "http-request" - next() +app.use(function(req, res, next){ + Metrics.inc("http-request"); + return next(); +}); -app.use (req, res, next) -> - requestDomain = domain.create() - requestDomain.add req - requestDomain.add res - requestDomain.on "error", (err)-> - try - # request a shutdown to prevent memory leaks - beginShutdown() - if !res.headerSent - res.send(500, "uncaught exception") - logger = require('logger-sharelatex') - req = - body:req.body - headers:req.headers - url:req.url - key: req.key +app.use(function(req, res, next) { + const requestDomain = domain.create(); + requestDomain.add(req); + requestDomain.add(res); + requestDomain.on("error", function(err){ + try { + // request a shutdown to prevent memory leaks + beginShutdown(); + if (!res.headerSent) { + res.send(500, "uncaught exception"); + } + logger = require('logger-sharelatex'); + req = { + body:req.body, + headers:req.headers, + url:req.url, + key: req.key, statusCode: req.statusCode - err = - message: err.message - stack: err.stack - name: err.name - type: err.type + }; + err = { + message: err.message, + stack: err.stack, + name: err.name, + type: err.type, arguments: err.arguments - logger.err err:err, req:req, res:res, "uncaught exception thrown on request" - catch exception - logger.err err: exception, "exception in request domain handler" - requestDomain.run next + }; + return logger.err({err, req, res}, "uncaught exception thrown on request"); + } catch (exception) { + return logger.err({err: exception}, "exception in request domain handler"); + } + }); + return requestDomain.run(next); +}); -app.use (req, res, next) -> - if not appIsOk - # when shutting down, close any HTTP keep-alive connections - res.set 'Connection', 'close' - next() +app.use(function(req, res, next) { + if (!appIsOk) { + // when shutting down, close any HTTP keep-alive connections + res.set('Connection', 'close'); + } + return next(); +}); -Metrics.injectMetricsRoute(app) +Metrics.injectMetricsRoute(app); -app.head "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.getFileHead -app.get "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.getFile -app.post "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.insertFile -app.put "/project/:project_id/file/:file_id", keyBuilder.userFileKey, bodyParser.json(), fileController.copyFile -app.del "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.deleteFile +app.head("/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.getFileHead); +app.get("/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.getFile); +app.post("/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.insertFile); +app.put("/project/:project_id/file/:file_id", keyBuilder.userFileKey, bodyParser.json(), fileController.copyFile); +app.del("/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.deleteFile); -app.head "/template/:template_id/v/:version/:format", keyBuilder.templateFileKey, fileController.getFileHead -app.get "/template/:template_id/v/:version/:format", keyBuilder.templateFileKey, fileController.getFile -app.get "/template/:template_id/v/:version/:format/:sub_type", keyBuilder.templateFileKey, fileController.getFile -app.post "/template/:template_id/v/:version/:format", keyBuilder.templateFileKey, fileController.insertFile +app.head("/template/:template_id/v/:version/:format", keyBuilder.templateFileKey, fileController.getFileHead); +app.get("/template/:template_id/v/:version/:format", keyBuilder.templateFileKey, fileController.getFile); +app.get("/template/:template_id/v/:version/:format/:sub_type", keyBuilder.templateFileKey, fileController.getFile); +app.post("/template/:template_id/v/:version/:format", keyBuilder.templateFileKey, fileController.insertFile); -app.head "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.getFileHead -app.get "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.getFile -app.post "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.insertFile -app.put "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, bodyParser.json(), fileController.copyFile -app.del "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.deleteFile +app.head("/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.getFileHead); +app.get("/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.getFile); +app.post("/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.insertFile); +app.put("/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, bodyParser.json(), fileController.copyFile); +app.del("/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.deleteFile); -app.get "/project/:project_id/size", keyBuilder.publicProjectKey, fileController.directorySize +app.get("/project/:project_id/size", keyBuilder.publicProjectKey, fileController.directorySize); -app.get "/bucket/:bucket/key/*", bucketController.getFile +app.get("/bucket/:bucket/key/*", bucketController.getFile); -app.get "/heapdump", (req, res)-> - require('heapdump').writeSnapshot '/tmp/' + Date.now() + '.filestore.heapsnapshot', (err, filename)-> - res.send filename +app.get("/heapdump", (req, res) => require('heapdump').writeSnapshot('/tmp/' + Date.now() + '.filestore.heapsnapshot', (err, filename) => res.send(filename))); -app.post "/shutdown", (req, res)-> - appIsOk = false - res.send() +app.post("/shutdown", function(req, res){ + appIsOk = false; + return res.send(); +}); -app.get '/status', (req, res)-> - if appIsOk - res.send('filestore sharelatex up') - else - logger.log "app is not ok - shutting down" - res.send("server is being shut down", 500) +app.get('/status', function(req, res){ + if (appIsOk) { + return res.send('filestore sharelatex up'); + } else { + logger.log("app is not ok - shutting down"); + return res.send("server is being shut down", 500); + } +}); -app.get "/health_check", healthCheckController.check +app.get("/health_check", healthCheckController.check); -app.get '*', (req, res)-> - res.send 404 +app.get('*', (req, res) => res.send(404)); -beginShutdown = () -> - if appIsOk - appIsOk = false - # hard-terminate this process if graceful shutdown fails - killTimer = setTimeout () -> - process.exit 1 - , 120*1000 - killTimer.unref?() # prevent timer from keeping process alive - server.close () -> - logger.log "closed all connections" - Metrics.close() - process.disconnect?() - logger.log "server will stop accepting connections" +var beginShutdown = function() { + if (appIsOk) { + appIsOk = false; + // hard-terminate this process if graceful shutdown fails + const killTimer = setTimeout(() => process.exit(1) + , 120*1000); + if (typeof killTimer.unref === 'function') { + killTimer.unref(); + } // prevent timer from keeping process alive + server.close(function() { + logger.log("closed all connections"); + Metrics.close(); + return (typeof process.disconnect === 'function' ? process.disconnect() : undefined); + }); + return logger.log("server will stop accepting connections"); + } +}; -port = settings.internal.filestore.port or 3009 -host = "0.0.0.0" +const port = settings.internal.filestore.port || 3009; +const host = "0.0.0.0"; -if !module.parent # Called directly - server = app.listen port, host, (error) -> - logger.info "Filestore starting up, listening on #{host}:#{port}" +if (!module.parent) { // Called directly + var server = app.listen(port, host, error => logger.info(`Filestore starting up, listening on ${host}:${port}`)); +} -module.exports = app +module.exports = app; -process.on 'SIGTERM', () -> - logger.log("filestore got SIGTERM, shutting down gracefully") - beginShutdown() +process.on('SIGTERM', function() { + logger.log("filestore got SIGTERM, shutting down gracefully"); + return beginShutdown(); +}); -if global.gc? - gcTimer = setInterval () -> - global.gc() - logger.log process.memoryUsage(), "global.gc" - , 3 * oneMinute = 60 * 1000 - gcTimer.unref() +if (global.gc != null) { + let oneMinute; + const gcTimer = setInterval(function() { + global.gc(); + return logger.log(process.memoryUsage(), "global.gc"); + } + , 3 * (oneMinute = 60 * 1000)); + gcTimer.unref(); +} From 9943e2cc8f198b05661fd2c4a095816db22059c2 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 16 Dec 2019 10:52:42 +0000 Subject: [PATCH 19/73] Prettier: convert individual decaffeinated files to Prettier format --- services/filestore/app.js | 336 +++++++++++++++++++++++--------------- 1 file changed, 202 insertions(+), 134 deletions(-) diff --git a/services/filestore/app.js b/services/filestore/app.js index 8a5bd9a04a..cbe13e8076 100644 --- a/services/filestore/app.js +++ b/services/filestore/app.js @@ -4,170 +4,238 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const Metrics = require("metrics-sharelatex"); -Metrics.initialize("filestore"); -const express = require('express'); -const bodyParser = require("body-parser"); -let logger = require('logger-sharelatex'); -logger.initialize("filestore"); -const settings = require("settings-sharelatex"); -const request = require("request"); -const fileController = require("./app/js/FileController"); -const bucketController = require("./app/js/BucketController"); -const keyBuilder = require("./app/js/KeyBuilder"); -const healthCheckController = require("./app/js/HealthCheckController"); -const domain = require("domain"); -let appIsOk = true; -const app = express(); +const Metrics = require('metrics-sharelatex') +Metrics.initialize('filestore') +const express = require('express') +const bodyParser = require('body-parser') +let logger = require('logger-sharelatex') +logger.initialize('filestore') +const settings = require('settings-sharelatex') +const request = require('request') +const fileController = require('./app/js/FileController') +const bucketController = require('./app/js/BucketController') +const keyBuilder = require('./app/js/KeyBuilder') +const healthCheckController = require('./app/js/HealthCheckController') +const domain = require('domain') +let appIsOk = true +const app = express() if ((settings.sentry != null ? settings.sentry.dsn : undefined) != null) { - logger.initializeErrorReporting(settings.sentry.dsn); + logger.initializeErrorReporting(settings.sentry.dsn) } -Metrics.open_sockets.monitor(logger); +Metrics.open_sockets.monitor(logger) if (Metrics.event_loop != null) { - Metrics.event_loop.monitor(logger); + Metrics.event_loop.monitor(logger) } -Metrics.memory.monitor(logger); +Metrics.memory.monitor(logger) -app.use(Metrics.http.monitor(logger)); - -app.use(function(req, res, next){ - Metrics.inc("http-request"); - return next(); -}); +app.use(Metrics.http.monitor(logger)) app.use(function(req, res, next) { - const requestDomain = domain.create(); - requestDomain.add(req); - requestDomain.add(res); - requestDomain.on("error", function(err){ - try { - // request a shutdown to prevent memory leaks - beginShutdown(); - if (!res.headerSent) { - res.send(500, "uncaught exception"); - } - logger = require('logger-sharelatex'); - req = { - body:req.body, - headers:req.headers, - url:req.url, - key: req.key, - statusCode: req.statusCode - }; - err = { - message: err.message, - stack: err.stack, - name: err.name, - type: err.type, - arguments: err.arguments - }; - return logger.err({err, req, res}, "uncaught exception thrown on request"); - } catch (exception) { - return logger.err({err: exception}, "exception in request domain handler"); - } - }); - return requestDomain.run(next); -}); + Metrics.inc('http-request') + return next() +}) app.use(function(req, res, next) { - if (!appIsOk) { - // when shutting down, close any HTTP keep-alive connections - res.set('Connection', 'close'); - } - return next(); -}); + const requestDomain = domain.create() + requestDomain.add(req) + requestDomain.add(res) + requestDomain.on('error', function(err) { + try { + // request a shutdown to prevent memory leaks + beginShutdown() + if (!res.headerSent) { + res.send(500, 'uncaught exception') + } + logger = require('logger-sharelatex') + req = { + body: req.body, + headers: req.headers, + url: req.url, + key: req.key, + statusCode: req.statusCode + } + err = { + message: err.message, + stack: err.stack, + name: err.name, + type: err.type, + arguments: err.arguments + } + return logger.err( + { err, req, res }, + 'uncaught exception thrown on request' + ) + } catch (exception) { + return logger.err( + { err: exception }, + 'exception in request domain handler' + ) + } + }) + return requestDomain.run(next) +}) -Metrics.injectMetricsRoute(app); +app.use(function(req, res, next) { + if (!appIsOk) { + // when shutting down, close any HTTP keep-alive connections + res.set('Connection', 'close') + } + return next() +}) -app.head("/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.getFileHead); -app.get("/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.getFile); -app.post("/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.insertFile); -app.put("/project/:project_id/file/:file_id", keyBuilder.userFileKey, bodyParser.json(), fileController.copyFile); -app.del("/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.deleteFile); +Metrics.injectMetricsRoute(app) -app.head("/template/:template_id/v/:version/:format", keyBuilder.templateFileKey, fileController.getFileHead); -app.get("/template/:template_id/v/:version/:format", keyBuilder.templateFileKey, fileController.getFile); -app.get("/template/:template_id/v/:version/:format/:sub_type", keyBuilder.templateFileKey, fileController.getFile); -app.post("/template/:template_id/v/:version/:format", keyBuilder.templateFileKey, fileController.insertFile); +app.head( + '/project/:project_id/file/:file_id', + keyBuilder.userFileKey, + fileController.getFileHead +) +app.get( + '/project/:project_id/file/:file_id', + keyBuilder.userFileKey, + fileController.getFile +) +app.post( + '/project/:project_id/file/:file_id', + keyBuilder.userFileKey, + fileController.insertFile +) +app.put( + '/project/:project_id/file/:file_id', + keyBuilder.userFileKey, + bodyParser.json(), + fileController.copyFile +) +app.del( + '/project/:project_id/file/:file_id', + keyBuilder.userFileKey, + fileController.deleteFile +) +app.head( + '/template/:template_id/v/:version/:format', + keyBuilder.templateFileKey, + fileController.getFileHead +) +app.get( + '/template/:template_id/v/:version/:format', + keyBuilder.templateFileKey, + fileController.getFile +) +app.get( + '/template/:template_id/v/:version/:format/:sub_type', + keyBuilder.templateFileKey, + fileController.getFile +) +app.post( + '/template/:template_id/v/:version/:format', + keyBuilder.templateFileKey, + fileController.insertFile +) -app.head("/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.getFileHead); -app.get("/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.getFile); -app.post("/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.insertFile); -app.put("/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, bodyParser.json(), fileController.copyFile); -app.del("/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.deleteFile); +app.head( + '/project/:project_id/public/:public_file_id', + keyBuilder.publicFileKey, + fileController.getFileHead +) +app.get( + '/project/:project_id/public/:public_file_id', + keyBuilder.publicFileKey, + fileController.getFile +) +app.post( + '/project/:project_id/public/:public_file_id', + keyBuilder.publicFileKey, + fileController.insertFile +) +app.put( + '/project/:project_id/public/:public_file_id', + keyBuilder.publicFileKey, + bodyParser.json(), + fileController.copyFile +) +app.del( + '/project/:project_id/public/:public_file_id', + keyBuilder.publicFileKey, + fileController.deleteFile +) -app.get("/project/:project_id/size", keyBuilder.publicProjectKey, fileController.directorySize); +app.get( + '/project/:project_id/size', + keyBuilder.publicProjectKey, + fileController.directorySize +) -app.get("/bucket/:bucket/key/*", bucketController.getFile); +app.get('/bucket/:bucket/key/*', bucketController.getFile) -app.get("/heapdump", (req, res) => require('heapdump').writeSnapshot('/tmp/' + Date.now() + '.filestore.heapsnapshot', (err, filename) => res.send(filename))); +app.get('/heapdump', (req, res) => + require('heapdump').writeSnapshot( + '/tmp/' + Date.now() + '.filestore.heapsnapshot', + (err, filename) => res.send(filename) + ) +) -app.post("/shutdown", function(req, res){ - appIsOk = false; - return res.send(); -}); - -app.get('/status', function(req, res){ - if (appIsOk) { - return res.send('filestore sharelatex up'); - } else { - logger.log("app is not ok - shutting down"); - return res.send("server is being shut down", 500); - } -}); - - -app.get("/health_check", healthCheckController.check); - - -app.get('*', (req, res) => res.send(404)); +app.post('/shutdown', function(req, res) { + appIsOk = false + return res.send() +}) +app.get('/status', function(req, res) { + if (appIsOk) { + return res.send('filestore sharelatex up') + } else { + logger.log('app is not ok - shutting down') + return res.send('server is being shut down', 500) + } +}) +app.get('/health_check', healthCheckController.check) +app.get('*', (req, res) => res.send(404)) var beginShutdown = function() { - if (appIsOk) { - appIsOk = false; - // hard-terminate this process if graceful shutdown fails - const killTimer = setTimeout(() => process.exit(1) - , 120*1000); - if (typeof killTimer.unref === 'function') { - killTimer.unref(); - } // prevent timer from keeping process alive - server.close(function() { - logger.log("closed all connections"); - Metrics.close(); - return (typeof process.disconnect === 'function' ? process.disconnect() : undefined); - }); - return logger.log("server will stop accepting connections"); - } -}; - - -const port = settings.internal.filestore.port || 3009; -const host = "0.0.0.0"; - -if (!module.parent) { // Called directly - var server = app.listen(port, host, error => logger.info(`Filestore starting up, listening on ${host}:${port}`)); + if (appIsOk) { + appIsOk = false + // hard-terminate this process if graceful shutdown fails + const killTimer = setTimeout(() => process.exit(1), 120 * 1000) + if (typeof killTimer.unref === 'function') { + killTimer.unref() + } // prevent timer from keeping process alive + server.close(function() { + logger.log('closed all connections') + Metrics.close() + return typeof process.disconnect === 'function' + ? process.disconnect() + : undefined + }) + return logger.log('server will stop accepting connections') + } } +const port = settings.internal.filestore.port || 3009 +const host = '0.0.0.0' -module.exports = app; +if (!module.parent) { + // Called directly + var server = app.listen(port, host, error => + logger.info(`Filestore starting up, listening on ${host}:${port}`) + ) +} + +module.exports = app process.on('SIGTERM', function() { - logger.log("filestore got SIGTERM, shutting down gracefully"); - return beginShutdown(); -}); + logger.log('filestore got SIGTERM, shutting down gracefully') + return beginShutdown() +}) if (global.gc != null) { - let oneMinute; - const gcTimer = setInterval(function() { - global.gc(); - return logger.log(process.memoryUsage(), "global.gc"); - } - , 3 * (oneMinute = 60 * 1000)); - gcTimer.unref(); + let oneMinute + const gcTimer = setInterval(function() { + global.gc() + return logger.log(process.memoryUsage(), 'global.gc') + }, 3 * (oneMinute = 60 * 1000)) + gcTimer.unref() } From 0b6e725137e3cb5996843acaee06a74869fef398 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 16 Dec 2019 11:16:27 +0000 Subject: [PATCH 20/73] Remove coffeescript compiles from package.json --- services/filestore/package.json | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/services/filestore/package.json b/services/filestore/package.json index 708656e0d6..40aabd8279 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -8,18 +8,13 @@ }, "scripts": { "test:acceptance:run": "mocha --recursive --reporter spec --timeout 15000 $@ test/acceptance/js", - "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP", + "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP", "test:unit:run": "mocha --recursive --reporter spec $@ test/unit/js", - "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP", - "compile:unit_tests": "[ ! -e test/unit/coffee ] && echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee", - "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee", - "compile:app": "([ -e app/coffee ] && coffee -m $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee -m $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')", - "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests && npm run compile:smoke_tests", - "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js", + "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", + "start": "node $NODE_APP_OPTIONS app.js", "nodemon": "nodemon --config nodemon.json", "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js", - "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js", - "compile:smoke_tests": "[ ! -e test/smoke/coffee ] && echo 'No smoke tests to compile' || coffee -o test/smoke/js -c test/smoke/coffee" + "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js" }, "dependencies": { "async": "~0.2.10", From e06c03e536b4b28acd6afc9d8d7ecc77d966cfa5 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 16 Dec 2019 11:16:37 +0000 Subject: [PATCH 21/73] Fix decaf error in FSPersistorManager --- services/filestore/app/js/FSPersistorManager.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/services/filestore/app/js/FSPersistorManager.js b/services/filestore/app/js/FSPersistorManager.js index 2e0de65632..d11d839df7 100644 --- a/services/filestore/app/js/FSPersistorManager.js +++ b/services/filestore/app/js/FSPersistorManager.js @@ -82,11 +82,11 @@ module.exports = { { err, location, filteredName: name }, 'Error reading from file' ) - } - if (err.code === 'ENOENT') { - return callback(new Errors.NotFoundError(err.message), null) - } else { - return callback(err, null) + if (err.code === 'ENOENT') { + return callback(new Errors.NotFoundError(err.message), null) + } else { + return callback(err, null) + } } opts.fd = fd const sourceStream = fs.createReadStream(null, opts) From 72c83bdafff276a68e742e100ecbdb6c2a41e3fb Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Mon, 16 Dec 2019 11:20:20 +0000 Subject: [PATCH 22/73] decaffeinate: Rename AWSSDKPersistorManagerTests.coffee and 12 other files from .coffee to .js --- ...ersistorManagerTests.coffee => AWSSDKPersistorManagerTests.js} | 0 .../{BucketControllerTests.coffee => BucketControllerTests.js} | 0 ...{FSPersistorManagerTests.coffee => FSPersistorManagerTests.js} | 0 .../coffee/{FileControllerTests.coffee => FileControllerTests.js} | 0 .../coffee/{FileConverterTests.coffee => FileConverterTests.js} | 0 .../unit/coffee/{FileHandlerTests.coffee => FileHandlerTests.js} | 0 .../coffee/{ImageOptimiserTests.coffee => ImageOptimiserTests.js} | 0 .../unit/coffee/{KeybuilderTests.coffee => KeybuilderTests.js} | 0 .../{LocalFileWriterTests.coffee => LocalFileWriterTests.js} | 0 .../{PersistorManagerTests.coffee => PersistorManagerTests.js} | 0 ...{S3PersistorManagerTests.coffee => S3PersistorManagerTests.js} | 0 .../test/unit/coffee/{SafeExecTests.coffee => SafeExecTests.js} | 0 .../test/unit/coffee/{SettingsTests.coffee => SettingsTests.js} | 0 13 files changed, 0 insertions(+), 0 deletions(-) rename services/filestore/test/unit/coffee/{AWSSDKPersistorManagerTests.coffee => AWSSDKPersistorManagerTests.js} (100%) rename services/filestore/test/unit/coffee/{BucketControllerTests.coffee => BucketControllerTests.js} (100%) rename services/filestore/test/unit/coffee/{FSPersistorManagerTests.coffee => FSPersistorManagerTests.js} (100%) rename services/filestore/test/unit/coffee/{FileControllerTests.coffee => FileControllerTests.js} (100%) rename services/filestore/test/unit/coffee/{FileConverterTests.coffee => FileConverterTests.js} (100%) rename services/filestore/test/unit/coffee/{FileHandlerTests.coffee => FileHandlerTests.js} (100%) rename services/filestore/test/unit/coffee/{ImageOptimiserTests.coffee => ImageOptimiserTests.js} (100%) rename services/filestore/test/unit/coffee/{KeybuilderTests.coffee => KeybuilderTests.js} (100%) rename services/filestore/test/unit/coffee/{LocalFileWriterTests.coffee => LocalFileWriterTests.js} (100%) rename services/filestore/test/unit/coffee/{PersistorManagerTests.coffee => PersistorManagerTests.js} (100%) rename services/filestore/test/unit/coffee/{S3PersistorManagerTests.coffee => S3PersistorManagerTests.js} (100%) rename services/filestore/test/unit/coffee/{SafeExecTests.coffee => SafeExecTests.js} (100%) rename services/filestore/test/unit/coffee/{SettingsTests.coffee => SettingsTests.js} (100%) diff --git a/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.coffee b/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.js similarity index 100% rename from services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.coffee rename to services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.js diff --git a/services/filestore/test/unit/coffee/BucketControllerTests.coffee b/services/filestore/test/unit/coffee/BucketControllerTests.js similarity index 100% rename from services/filestore/test/unit/coffee/BucketControllerTests.coffee rename to services/filestore/test/unit/coffee/BucketControllerTests.js diff --git a/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee b/services/filestore/test/unit/coffee/FSPersistorManagerTests.js similarity index 100% rename from services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee rename to services/filestore/test/unit/coffee/FSPersistorManagerTests.js diff --git a/services/filestore/test/unit/coffee/FileControllerTests.coffee b/services/filestore/test/unit/coffee/FileControllerTests.js similarity index 100% rename from services/filestore/test/unit/coffee/FileControllerTests.coffee rename to services/filestore/test/unit/coffee/FileControllerTests.js diff --git a/services/filestore/test/unit/coffee/FileConverterTests.coffee b/services/filestore/test/unit/coffee/FileConverterTests.js similarity index 100% rename from services/filestore/test/unit/coffee/FileConverterTests.coffee rename to services/filestore/test/unit/coffee/FileConverterTests.js diff --git a/services/filestore/test/unit/coffee/FileHandlerTests.coffee b/services/filestore/test/unit/coffee/FileHandlerTests.js similarity index 100% rename from services/filestore/test/unit/coffee/FileHandlerTests.coffee rename to services/filestore/test/unit/coffee/FileHandlerTests.js diff --git a/services/filestore/test/unit/coffee/ImageOptimiserTests.coffee b/services/filestore/test/unit/coffee/ImageOptimiserTests.js similarity index 100% rename from services/filestore/test/unit/coffee/ImageOptimiserTests.coffee rename to services/filestore/test/unit/coffee/ImageOptimiserTests.js diff --git a/services/filestore/test/unit/coffee/KeybuilderTests.coffee b/services/filestore/test/unit/coffee/KeybuilderTests.js similarity index 100% rename from services/filestore/test/unit/coffee/KeybuilderTests.coffee rename to services/filestore/test/unit/coffee/KeybuilderTests.js diff --git a/services/filestore/test/unit/coffee/LocalFileWriterTests.coffee b/services/filestore/test/unit/coffee/LocalFileWriterTests.js similarity index 100% rename from services/filestore/test/unit/coffee/LocalFileWriterTests.coffee rename to services/filestore/test/unit/coffee/LocalFileWriterTests.js diff --git a/services/filestore/test/unit/coffee/PersistorManagerTests.coffee b/services/filestore/test/unit/coffee/PersistorManagerTests.js similarity index 100% rename from services/filestore/test/unit/coffee/PersistorManagerTests.coffee rename to services/filestore/test/unit/coffee/PersistorManagerTests.js diff --git a/services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee b/services/filestore/test/unit/coffee/S3PersistorManagerTests.js similarity index 100% rename from services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee rename to services/filestore/test/unit/coffee/S3PersistorManagerTests.js diff --git a/services/filestore/test/unit/coffee/SafeExecTests.coffee b/services/filestore/test/unit/coffee/SafeExecTests.js similarity index 100% rename from services/filestore/test/unit/coffee/SafeExecTests.coffee rename to services/filestore/test/unit/coffee/SafeExecTests.js diff --git a/services/filestore/test/unit/coffee/SettingsTests.coffee b/services/filestore/test/unit/coffee/SettingsTests.js similarity index 100% rename from services/filestore/test/unit/coffee/SettingsTests.coffee rename to services/filestore/test/unit/coffee/SettingsTests.js From 5cf7138ef1b9fb0d7883122de863467c8b669189 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Mon, 16 Dec 2019 11:20:22 +0000 Subject: [PATCH 23/73] decaffeinate: Convert AWSSDKPersistorManagerTests.coffee and 12 other files to JS --- .../coffee/AWSSDKPersistorManagerTests.js | 541 ++++++++------ .../test/unit/coffee/BucketControllerTests.js | 147 ++-- .../unit/coffee/FSPersistorManagerTests.js | 555 ++++++++------ .../test/unit/coffee/FileControllerTests.js | 419 ++++++----- .../test/unit/coffee/FileConverterTests.js | 167 +++-- .../test/unit/coffee/FileHandlerTests.js | 461 +++++++----- .../test/unit/coffee/ImageOptimiserTests.js | 124 ++-- .../test/unit/coffee/KeybuilderTests.js | 84 ++- .../test/unit/coffee/LocalFileWriterTests.js | 174 +++-- .../test/unit/coffee/PersistorManagerTests.js | 200 ++--- .../unit/coffee/S3PersistorManagerTests.js | 696 ++++++++++-------- .../test/unit/coffee/SafeExecTests.js | 109 +-- .../test/unit/coffee/SettingsTests.js | 44 +- 13 files changed, 2167 insertions(+), 1554 deletions(-) diff --git a/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.js b/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.js index 92fb968914..cad0f4b805 100644 --- a/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.js +++ b/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.js @@ -1,272 +1,353 @@ -sinon = require 'sinon' -chai = require 'chai' +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const sinon = require('sinon'); +const chai = require('chai'); -should = chai.should() -expect = chai.expect +const should = chai.should(); +const { + expect +} = chai; -modulePath = "../../../app/js/AWSSDKPersistorManager.js" -SandboxedModule = require 'sandboxed-module' +const modulePath = "../../../app/js/AWSSDKPersistorManager.js"; +const SandboxedModule = require('sandboxed-module'); -describe "AWSSDKPersistorManager", -> - beforeEach -> - @settings = - filestore: +describe("AWSSDKPersistorManager", function() { + beforeEach(function() { + this.settings = { + filestore: { backend: "aws-sdk" - @s3 = - upload: sinon.stub() - getObject: sinon.stub() - copyObject: sinon.stub() - deleteObject: sinon.stub() - listObjects: sinon.stub() - deleteObjects: sinon.stub() + } + }; + this.s3 = { + upload: sinon.stub(), + getObject: sinon.stub(), + copyObject: sinon.stub(), + deleteObject: sinon.stub(), + listObjects: sinon.stub(), + deleteObjects: sinon.stub(), headObject: sinon.stub() - @awssdk = - S3: sinon.stub().returns @s3 + }; + this.awssdk = + {S3: sinon.stub().returns(this.s3)}; - @requires = - "aws-sdk": @awssdk - "settings-sharelatex": @settings - "logger-sharelatex": - log:-> - err:-> - "fs": @fs = - createReadStream: sinon.stub() - "./Errors": @Errors = - NotFoundError: sinon.stub() - @key = "my/key" - @bucketName = "my-bucket" - @error = "my error" - @AWSSDKPersistorManager = SandboxedModule.require modulePath, requires: @requires + this.requires = { + "aws-sdk": this.awssdk, + "settings-sharelatex": this.settings, + "logger-sharelatex": { + log() {}, + err() {} + }, + "fs": (this.fs = + {createReadStream: sinon.stub()}), + "./Errors": (this.Errors = + {NotFoundError: sinon.stub()}) + }; + this.key = "my/key"; + this.bucketName = "my-bucket"; + this.error = "my error"; + return this.AWSSDKPersistorManager = SandboxedModule.require(modulePath, {requires: this.requires}); + }); - describe "sendFile", -> - beforeEach -> - @stream = {} - @fsPath = "/usr/local/some/file" - @fs.createReadStream.returns @stream + describe("sendFile", function() { + beforeEach(function() { + this.stream = {}; + this.fsPath = "/usr/local/some/file"; + return this.fs.createReadStream.returns(this.stream); + }); - it "should put the file with s3.upload", (done) -> - @s3.upload.callsArgWith 1 - @AWSSDKPersistorManager.sendFile @bucketName, @key, @fsPath, (err) => - expect(err).to.not.be.ok - expect(@s3.upload.calledOnce, "called only once").to.be.true - expect((@s3.upload.calledWith Bucket: @bucketName, Key: @key, Body: @stream) - , "called with correct arguments").to.be.true - done() + it("should put the file with s3.upload", function(done) { + this.s3.upload.callsArgWith(1); + return this.AWSSDKPersistorManager.sendFile(this.bucketName, this.key, this.fsPath, err => { + expect(err).to.not.be.ok; + expect(this.s3.upload.calledOnce, "called only once").to.be.true; + expect((this.s3.upload.calledWith({Bucket: this.bucketName, Key: this.key, Body: this.stream})) + , "called with correct arguments").to.be.true; + return done(); + }); + }); - it "should dispatch the error from s3.upload", (done) -> - @s3.upload.callsArgWith 1, @error - @AWSSDKPersistorManager.sendFile @bucketName, @key, @fsPath, (err) => - expect(err).to.equal @error - done() + return it("should dispatch the error from s3.upload", function(done) { + this.s3.upload.callsArgWith(1, this.error); + return this.AWSSDKPersistorManager.sendFile(this.bucketName, this.key, this.fsPath, err => { + expect(err).to.equal(this.error); + return done(); + }); + }); + }); - describe "sendStream", -> - beforeEach -> - @stream = {} + describe("sendStream", function() { + beforeEach(function() { + return this.stream = {};}); - it "should put the file with s3.upload", (done) -> - @s3.upload.callsArgWith 1 - @AWSSDKPersistorManager.sendStream @bucketName, @key, @stream, (err) => - expect(err).to.not.be.ok - expect(@s3.upload.calledOnce, "called only once").to.be.true - expect((@s3.upload.calledWith Bucket: @bucketName, Key: @key, Body: @stream), - "called with correct arguments").to.be.true - done() + it("should put the file with s3.upload", function(done) { + this.s3.upload.callsArgWith(1); + return this.AWSSDKPersistorManager.sendStream(this.bucketName, this.key, this.stream, err => { + expect(err).to.not.be.ok; + expect(this.s3.upload.calledOnce, "called only once").to.be.true; + expect((this.s3.upload.calledWith({Bucket: this.bucketName, Key: this.key, Body: this.stream})), + "called with correct arguments").to.be.true; + return done(); + }); + }); - it "should dispatch the error from s3.upload", (done) -> - @s3.upload.callsArgWith 1, @error - @AWSSDKPersistorManager.sendStream @bucketName, @key, @stream, (err) => - expect(err).to.equal @error - done() + return it("should dispatch the error from s3.upload", function(done) { + this.s3.upload.callsArgWith(1, this.error); + return this.AWSSDKPersistorManager.sendStream(this.bucketName, this.key, this.stream, err => { + expect(err).to.equal(this.error); + return done(); + }); + }); + }); - describe "getFileStream", -> - beforeEach -> - @opts = {} - @stream = {} - @read_stream = - on: @read_stream_on = sinon.stub() - @object = - createReadStream: sinon.stub().returns @read_stream - @s3.getObject.returns @object + describe("getFileStream", function() { + beforeEach(function() { + this.opts = {}; + this.stream = {}; + this.read_stream = + {on: (this.read_stream_on = sinon.stub())}; + this.object = + {createReadStream: sinon.stub().returns(this.read_stream)}; + return this.s3.getObject.returns(this.object); + }); - it "should return a stream from s3.getObject", (done) -> - @read_stream_on.withArgs('readable').callsArgWith 1 + it("should return a stream from s3.getObject", function(done) { + this.read_stream_on.withArgs('readable').callsArgWith(1); - @AWSSDKPersistorManager.getFileStream @bucketName, @key, @opts, (err, stream) => - expect(@read_stream_on.calledTwice) - expect(err).to.not.be.ok - expect(stream, "returned the stream").to.equal @read_stream - expect((@s3.getObject.calledWith Bucket: @bucketName, Key: @key), - "called with correct arguments").to.be.true - done() + return this.AWSSDKPersistorManager.getFileStream(this.bucketName, this.key, this.opts, (err, stream) => { + expect(this.read_stream_on.calledTwice); + expect(err).to.not.be.ok; + expect(stream, "returned the stream").to.equal(this.read_stream); + expect((this.s3.getObject.calledWith({Bucket: this.bucketName, Key: this.key})), + "called with correct arguments").to.be.true; + return done(); + }); + }); - describe "with start and end options", -> - beforeEach -> - @opts = - start: 0 + describe("with start and end options", function() { + beforeEach(function() { + return this.opts = { + start: 0, end: 8 - it "should pass headers to the s3.GetObject", (done) -> - @read_stream_on.withArgs('readable').callsArgWith 1 - @AWSSDKPersistorManager.getFileStream @bucketName, @key, @opts, (err, stream) => - expect((@s3.getObject.calledWith Bucket: @bucketName, Key: @key, Range: 'bytes=0-8'), - "called with correct arguments").to.be.true - done() + }; + }); + return it("should pass headers to the s3.GetObject", function(done) { + this.read_stream_on.withArgs('readable').callsArgWith(1); + this.AWSSDKPersistorManager.getFileStream(this.bucketName, this.key, this.opts, (err, stream) => { + return expect((this.s3.getObject.calledWith({Bucket: this.bucketName, Key: this.key, Range: 'bytes=0-8'})), + "called with correct arguments").to.be.true; + }); + return done(); + }); + }); - describe "error conditions", -> - describe "when the file doesn't exist", -> - beforeEach -> - @error = new Error() - @error.code = 'NoSuchKey' - it "should produce a NotFoundError", (done) -> - @read_stream_on.withArgs('error').callsArgWith 1, @error - @AWSSDKPersistorManager.getFileStream @bucketName, @key, @opts, (err, stream) => - expect(stream).to.not.be.ok - expect(err).to.be.ok - expect(err instanceof @Errors.NotFoundError, "error is a correct instance").to.equal true - done() + return describe("error conditions", function() { + describe("when the file doesn't exist", function() { + beforeEach(function() { + this.error = new Error(); + return this.error.code = 'NoSuchKey'; + }); + return it("should produce a NotFoundError", function(done) { + this.read_stream_on.withArgs('error').callsArgWith(1, this.error); + return this.AWSSDKPersistorManager.getFileStream(this.bucketName, this.key, this.opts, (err, stream) => { + expect(stream).to.not.be.ok; + expect(err).to.be.ok; + expect(err instanceof this.Errors.NotFoundError, "error is a correct instance").to.equal(true); + return done(); + }); + }); + }); - describe "when there is some other error", -> - beforeEach -> - @error = new Error() - it "should dispatch the error from s3 object stream", (done) -> - @read_stream_on.withArgs('error').callsArgWith 1, @error - @AWSSDKPersistorManager.getFileStream @bucketName, @key, @opts, (err, stream) => - expect(stream).to.not.be.ok - expect(err).to.be.ok - expect(err).to.equal @error - done() + return describe("when there is some other error", function() { + beforeEach(function() { + return this.error = new Error(); + }); + return it("should dispatch the error from s3 object stream", function(done) { + this.read_stream_on.withArgs('error').callsArgWith(1, this.error); + return this.AWSSDKPersistorManager.getFileStream(this.bucketName, this.key, this.opts, (err, stream) => { + expect(stream).to.not.be.ok; + expect(err).to.be.ok; + expect(err).to.equal(this.error); + return done(); + }); + }); + }); + }); + }); - describe "copyFile", -> - beforeEach -> - @destKey = "some/key" - @stream = {} + describe("copyFile", function() { + beforeEach(function() { + this.destKey = "some/key"; + return this.stream = {};}); - it "should copy the file with s3.copyObject", (done) -> - @s3.copyObject.callsArgWith 1 - @AWSSDKPersistorManager.copyFile @bucketName, @key, @destKey, (err) => - expect(err).to.not.be.ok - expect(@s3.copyObject.calledOnce, "called only once").to.be.true - expect((@s3.copyObject.calledWith Bucket: @bucketName, Key: @destKey, CopySource: @bucketName + '/' + @key), - "called with correct arguments").to.be.true - done() + it("should copy the file with s3.copyObject", function(done) { + this.s3.copyObject.callsArgWith(1); + return this.AWSSDKPersistorManager.copyFile(this.bucketName, this.key, this.destKey, err => { + expect(err).to.not.be.ok; + expect(this.s3.copyObject.calledOnce, "called only once").to.be.true; + expect((this.s3.copyObject.calledWith({Bucket: this.bucketName, Key: this.destKey, CopySource: this.bucketName + '/' + this.key})), + "called with correct arguments").to.be.true; + return done(); + }); + }); - it "should dispatch the error from s3.copyObject", (done) -> - @s3.copyObject.callsArgWith 1, @error - @AWSSDKPersistorManager.copyFile @bucketName, @key, @destKey, (err) => - expect(err).to.equal @error - done() + return it("should dispatch the error from s3.copyObject", function(done) { + this.s3.copyObject.callsArgWith(1, this.error); + return this.AWSSDKPersistorManager.copyFile(this.bucketName, this.key, this.destKey, err => { + expect(err).to.equal(this.error); + return done(); + }); + }); + }); - describe "deleteFile", -> - it "should delete the file with s3.deleteObject", (done) -> - @s3.deleteObject.callsArgWith 1 - @AWSSDKPersistorManager.deleteFile @bucketName, @key, (err) => - expect(err).to.not.be.ok - expect(@s3.deleteObject.calledOnce, "called only once").to.be.true - expect((@s3.deleteObject.calledWith Bucket: @bucketName, Key: @key), - "called with correct arguments").to.be.true - done() + describe("deleteFile", function() { + it("should delete the file with s3.deleteObject", function(done) { + this.s3.deleteObject.callsArgWith(1); + return this.AWSSDKPersistorManager.deleteFile(this.bucketName, this.key, err => { + expect(err).to.not.be.ok; + expect(this.s3.deleteObject.calledOnce, "called only once").to.be.true; + expect((this.s3.deleteObject.calledWith({Bucket: this.bucketName, Key: this.key})), + "called with correct arguments").to.be.true; + return done(); + }); + }); - it "should dispatch the error from s3.deleteObject", (done) -> - @s3.deleteObject.callsArgWith 1, @error - @AWSSDKPersistorManager.deleteFile @bucketName, @key, (err) => - expect(err).to.equal @error - done() + return it("should dispatch the error from s3.deleteObject", function(done) { + this.s3.deleteObject.callsArgWith(1, this.error); + return this.AWSSDKPersistorManager.deleteFile(this.bucketName, this.key, err => { + expect(err).to.equal(this.error); + return done(); + }); + }); + }); - describe "deleteDirectory", -> + describe("deleteDirectory", function() { - it "should list the directory content using s3.listObjects", (done) -> - @s3.listObjects.callsArgWith 1, null, Contents: [] - @AWSSDKPersistorManager.deleteDirectory @bucketName, @key, (err) => - expect(err).to.not.be.ok - expect(@s3.listObjects.calledOnce, "called only once").to.be.true - expect((@s3.listObjects.calledWith Bucket: @bucketName, Prefix: @key), - "called with correct arguments").to.be.true - done() + it("should list the directory content using s3.listObjects", function(done) { + this.s3.listObjects.callsArgWith(1, null, {Contents: []}); + return this.AWSSDKPersistorManager.deleteDirectory(this.bucketName, this.key, err => { + expect(err).to.not.be.ok; + expect(this.s3.listObjects.calledOnce, "called only once").to.be.true; + expect((this.s3.listObjects.calledWith({Bucket: this.bucketName, Prefix: this.key})), + "called with correct arguments").to.be.true; + return done(); + }); + }); - it "should dispatch the error from s3.listObjects", (done) -> - @s3.listObjects.callsArgWith 1, @error - @AWSSDKPersistorManager.deleteDirectory @bucketName, @key, (err) => - expect(err).to.equal @error - done() + it("should dispatch the error from s3.listObjects", function(done) { + this.s3.listObjects.callsArgWith(1, this.error); + return this.AWSSDKPersistorManager.deleteDirectory(this.bucketName, this.key, err => { + expect(err).to.equal(this.error); + return done(); + }); + }); - describe "with directory content", -> - beforeEach -> - @fileList = [ - Key: 'foo' - , Key: 'bar' + return describe("with directory content", function() { + beforeEach(function() { + return this.fileList = [ + {Key: 'foo'} + , { Key: 'bar' , Key: 'baz' - ] + } + ];}); - it "should forward the file keys to s3.deleteObjects", (done) -> - @s3.listObjects.callsArgWith 1, null, Contents: @fileList - @s3.deleteObjects.callsArgWith 1 - @AWSSDKPersistorManager.deleteDirectory @bucketName, @key, (err) => - expect(err).to.not.be.ok - expect(@s3.deleteObjects.calledOnce, "called only once").to.be.true - expect((@s3.deleteObjects.calledWith - Bucket: @bucketName - Delete: - Quiet: true - Objects: @fileList), - "called with correct arguments").to.be.true - done() + it("should forward the file keys to s3.deleteObjects", function(done) { + this.s3.listObjects.callsArgWith(1, null, {Contents: this.fileList}); + this.s3.deleteObjects.callsArgWith(1); + return this.AWSSDKPersistorManager.deleteDirectory(this.bucketName, this.key, err => { + expect(err).to.not.be.ok; + expect(this.s3.deleteObjects.calledOnce, "called only once").to.be.true; + expect((this.s3.deleteObjects.calledWith({ + Bucket: this.bucketName, + Delete: { + Quiet: true, + Objects: this.fileList + }})), + "called with correct arguments").to.be.true; + return done(); + }); + }); - it "should dispatch the error from s3.deleteObjects", (done) -> - @s3.listObjects.callsArgWith 1, null, Contents: @fileList - @s3.deleteObjects.callsArgWith 1, @error - @AWSSDKPersistorManager.deleteDirectory @bucketName, @key, (err) => - expect(err).to.equal @error - done() + return it("should dispatch the error from s3.deleteObjects", function(done) { + this.s3.listObjects.callsArgWith(1, null, {Contents: this.fileList}); + this.s3.deleteObjects.callsArgWith(1, this.error); + return this.AWSSDKPersistorManager.deleteDirectory(this.bucketName, this.key, err => { + expect(err).to.equal(this.error); + return done(); + }); + }); + }); + }); - describe "checkIfFileExists", -> + describe("checkIfFileExists", function() { - it "should check for the file with s3.headObject", (done) -> - @s3.headObject.callsArgWith 1, null, {} - @AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) => - expect(err).to.not.be.ok - expect(@s3.headObject.calledOnce, "called only once").to.be.true - expect((@s3.headObject.calledWith Bucket: @bucketName, Key: @key), - "called with correct arguments").to.be.true - done() + it("should check for the file with s3.headObject", function(done) { + this.s3.headObject.callsArgWith(1, null, {}); + return this.AWSSDKPersistorManager.checkIfFileExists(this.bucketName, this.key, (err, exists) => { + expect(err).to.not.be.ok; + expect(this.s3.headObject.calledOnce, "called only once").to.be.true; + expect((this.s3.headObject.calledWith({Bucket: this.bucketName, Key: this.key})), + "called with correct arguments").to.be.true; + return done(); + }); + }); - it "should return false on an inexistant file", (done) -> - @s3.headObject.callsArgWith 1, null, {} - @AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) => - expect(exists).to.be.false - done() + it("should return false on an inexistant file", function(done) { + this.s3.headObject.callsArgWith(1, null, {}); + return this.AWSSDKPersistorManager.checkIfFileExists(this.bucketName, this.key, (err, exists) => { + expect(exists).to.be.false; + return done(); + }); + }); - it "should return true on an existing file", (done) -> - @s3.headObject.callsArgWith 1, null, ETag: "etag" - @AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) => - expect(exists).to.be.true - done() + it("should return true on an existing file", function(done) { + this.s3.headObject.callsArgWith(1, null, {ETag: "etag"}); + return this.AWSSDKPersistorManager.checkIfFileExists(this.bucketName, this.key, (err, exists) => { + expect(exists).to.be.true; + return done(); + }); + }); - it "should dispatch the error from s3.headObject", (done) -> - @s3.headObject.callsArgWith 1, @error - @AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) => - expect(err).to.equal @error - done() + return it("should dispatch the error from s3.headObject", function(done) { + this.s3.headObject.callsArgWith(1, this.error); + return this.AWSSDKPersistorManager.checkIfFileExists(this.bucketName, this.key, (err, exists) => { + expect(err).to.equal(this.error); + return done(); + }); + }); + }); - describe "directorySize", -> + return describe("directorySize", function() { - it "should list the directory content using s3.listObjects", (done) -> - @s3.listObjects.callsArgWith 1, null, Contents: [] - @AWSSDKPersistorManager.directorySize @bucketName, @key, (err) => - expect(err).to.not.be.ok - expect(@s3.listObjects.calledOnce, "called only once").to.be.true - expect((@s3.listObjects.calledWith Bucket: @bucketName, Prefix: @key), - "called with correct arguments").to.be.true - done() + it("should list the directory content using s3.listObjects", function(done) { + this.s3.listObjects.callsArgWith(1, null, {Contents: []}); + return this.AWSSDKPersistorManager.directorySize(this.bucketName, this.key, err => { + expect(err).to.not.be.ok; + expect(this.s3.listObjects.calledOnce, "called only once").to.be.true; + expect((this.s3.listObjects.calledWith({Bucket: this.bucketName, Prefix: this.key})), + "called with correct arguments").to.be.true; + return done(); + }); + }); - it "should dispatch the error from s3.listObjects", (done) -> - @s3.listObjects.callsArgWith 1, @error - @AWSSDKPersistorManager.directorySize @bucketName, @key, (err) => - expect(err).to.equal @error - done() + it("should dispatch the error from s3.listObjects", function(done) { + this.s3.listObjects.callsArgWith(1, this.error); + return this.AWSSDKPersistorManager.directorySize(this.bucketName, this.key, err => { + expect(err).to.equal(this.error); + return done(); + }); + }); - it "should sum directory files sizes", (done) -> - @s3.listObjects.callsArgWith 1, null, Contents: [ { Size: 1024 }, { Size: 2048 }] - @AWSSDKPersistorManager.directorySize @bucketName, @key, (err, size) => - expect(size).to.equal 3072 - done() + return it("should sum directory files sizes", function(done) { + this.s3.listObjects.callsArgWith(1, null, {Contents: [ { Size: 1024 }, { Size: 2048 }]}); + return this.AWSSDKPersistorManager.directorySize(this.bucketName, this.key, (err, size) => { + expect(size).to.equal(3072); + return done(); + }); + }); + }); +}); diff --git a/services/filestore/test/unit/coffee/BucketControllerTests.js b/services/filestore/test/unit/coffee/BucketControllerTests.js index c6bbabc89e..db0e6a5aa3 100644 --- a/services/filestore/test/unit/coffee/BucketControllerTests.js +++ b/services/filestore/test/unit/coffee/BucketControllerTests.js @@ -1,71 +1,100 @@ -assert = require("chai").assert -sinon = require('sinon') -chai = require('chai') -should = chai.should() -expect = chai.expect -modulePath = "../../../app/js/BucketController.js" -SandboxedModule = require('sandboxed-module') +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const { + assert +} = require("chai"); +const sinon = require('sinon'); +const chai = require('chai'); +const should = chai.should(); +const { + expect +} = chai; +const modulePath = "../../../app/js/BucketController.js"; +const SandboxedModule = require('sandboxed-module'); -describe "BucketController", -> +describe("BucketController", function() { - beforeEach -> - @PersistorManager = - sendStream: sinon.stub() - copyFile: sinon.stub() + beforeEach(function() { + this.PersistorManager = { + sendStream: sinon.stub(), + copyFile: sinon.stub(), deleteFile:sinon.stub() + }; - @settings = - s3: - buckets: + this.settings = { + s3: { + buckets: { user_files:"user_files" - filestore: - backend: "s3" - s3: - secret: "secret" + } + }, + filestore: { + backend: "s3", + s3: { + secret: "secret", key: "this_key" + } + } + }; - @FileHandler = - getFile: sinon.stub() - deleteFile: sinon.stub() - insertFile: sinon.stub() + this.FileHandler = { + getFile: sinon.stub(), + deleteFile: sinon.stub(), + insertFile: sinon.stub(), getDirectorySize: sinon.stub() - @LocalFileWriter = {} - @controller = SandboxedModule.require modulePath, requires: - "./LocalFileWriter":@LocalFileWriter - "./FileHandler": @FileHandler - "./PersistorManager":@PersistorManager - "settings-sharelatex": @settings - "metrics-sharelatex": - inc:-> - "logger-sharelatex": - log:-> - err:-> - @project_id = "project_id" - @file_id = "file_id" - @bucket = "user_files" - @key = "#{@project_id}/#{@file_id}" - @req = - query:{} - params: - bucket: @bucket - 0: @key + }; + this.LocalFileWriter = {}; + this.controller = SandboxedModule.require(modulePath, { requires: { + "./LocalFileWriter":this.LocalFileWriter, + "./FileHandler": this.FileHandler, + "./PersistorManager":this.PersistorManager, + "settings-sharelatex": this.settings, + "metrics-sharelatex": { + inc() {} + }, + "logger-sharelatex": { + log() {}, + err() {} + } + } + } + ); + this.project_id = "project_id"; + this.file_id = "file_id"; + this.bucket = "user_files"; + this.key = `${this.project_id}/${this.file_id}`; + this.req = { + query:{}, + params: { + bucket: this.bucket, + 0: this.key + }, headers: {} - @res = - setHeader: -> - @fileStream = {} + }; + this.res = + {setHeader() {}}; + return this.fileStream = {};}); - describe "getFile", -> + return describe("getFile", function() { - it "should pipe the stream", (done)-> - @FileHandler.getFile.callsArgWith(3, null, @fileStream) - @fileStream.pipe = (res)=> - res.should.equal @res - done() - @controller.getFile @req, @res + it("should pipe the stream", function(done){ + this.FileHandler.getFile.callsArgWith(3, null, this.fileStream); + this.fileStream.pipe = res=> { + res.should.equal(this.res); + return done(); + }; + return this.controller.getFile(this.req, this.res); + }); - it "should send a 500 if there is a problem", (done)-> - @FileHandler.getFile.callsArgWith(3, "error") - @res.send = (code)=> - code.should.equal 500 - done() - @controller.getFile @req, @res + return it("should send a 500 if there is a problem", function(done){ + this.FileHandler.getFile.callsArgWith(3, "error"); + this.res.send = code=> { + code.should.equal(500); + return done(); + }; + return this.controller.getFile(this.req, this.res); + }); + }); +}); diff --git a/services/filestore/test/unit/coffee/FSPersistorManagerTests.js b/services/filestore/test/unit/coffee/FSPersistorManagerTests.js index 6980ce3e30..cd73f41ac0 100644 --- a/services/filestore/test/unit/coffee/FSPersistorManagerTests.js +++ b/services/filestore/test/unit/coffee/FSPersistorManagerTests.js @@ -1,281 +1,372 @@ -assert = require("chai").assert -sinon = require('sinon') -chai = require('chai') -should = chai.should -expect = chai.expect -modulePath = "../../../app/js/FSPersistorManager.js" -SandboxedModule = require('sandboxed-module') -fs = require("fs") -response = require("response") +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const { + assert +} = require("chai"); +const sinon = require('sinon'); +const chai = require('chai'); +const { + should +} = chai; +const { + expect +} = chai; +const modulePath = "../../../app/js/FSPersistorManager.js"; +const SandboxedModule = require('sandboxed-module'); +const fs = require("fs"); +const response = require("response"); -describe "FSPersistorManagerTests", -> +describe("FSPersistorManagerTests", function() { - beforeEach -> - @Fs = - rename:sinon.stub() - createReadStream:sinon.stub() - createWriteStream:sinon.stub() - unlink:sinon.stub() - rmdir:sinon.stub() - exists:sinon.stub() - readdir:sinon.stub() - open:sinon.stub() - openSync:sinon.stub() - fstatSync:sinon.stub() - closeSync:sinon.stub() + beforeEach(function() { + this.Fs = { + rename:sinon.stub(), + createReadStream:sinon.stub(), + createWriteStream:sinon.stub(), + unlink:sinon.stub(), + rmdir:sinon.stub(), + exists:sinon.stub(), + readdir:sinon.stub(), + open:sinon.stub(), + openSync:sinon.stub(), + fstatSync:sinon.stub(), + closeSync:sinon.stub(), stat:sinon.stub() - @Rimraf = sinon.stub() - @LocalFileWriter = - writeStream: sinon.stub() + }; + this.Rimraf = sinon.stub(); + this.LocalFileWriter = { + writeStream: sinon.stub(), deleteFile: sinon.stub() - @requires = - "./LocalFileWriter":@LocalFileWriter - "fs":@Fs - "logger-sharelatex": - log:-> - err:-> - "response":response - "rimraf":@Rimraf - "./Errors": @Errors = - NotFoundError: sinon.stub() - @location = "/tmp" - @name1 = "530f2407e7ef165704000007/530f838b46d9a9e859000008" - @name1Filtered ="530f2407e7ef165704000007_530f838b46d9a9e859000008" - @name2 = "second_file" - @error = "error_message" - @FSPersistorManager = SandboxedModule.require modulePath, requires: @requires + }; + this.requires = { + "./LocalFileWriter":this.LocalFileWriter, + "fs":this.Fs, + "logger-sharelatex": { + log() {}, + err() {} + }, + "response":response, + "rimraf":this.Rimraf, + "./Errors": (this.Errors = + {NotFoundError: sinon.stub()}) + }; + this.location = "/tmp"; + this.name1 = "530f2407e7ef165704000007/530f838b46d9a9e859000008"; + this.name1Filtered ="530f2407e7ef165704000007_530f838b46d9a9e859000008"; + this.name2 = "second_file"; + this.error = "error_message"; + return this.FSPersistorManager = SandboxedModule.require(modulePath, {requires: this.requires}); + }); - describe "sendFile", -> - beforeEach -> - @Fs.createReadStream = sinon.stub().returns({ - on: -> - pipe: -> - }) + describe("sendFile", function() { + beforeEach(function() { + return this.Fs.createReadStream = sinon.stub().returns({ + on() {}, + pipe() {} + }); + }); - it "should copy the file", (done) -> - @Fs.createWriteStream =sinon.stub().returns({ - on: (event, handler) -> - process.nextTick(handler) if event is 'finish' - }) - @FSPersistorManager.sendFile @location, @name1, @name2, (err)=> - @Fs.createReadStream.calledWith(@name2).should.equal true - @Fs.createWriteStream.calledWith("#{@location}/#{@name1Filtered}" ).should.equal true - done() + it("should copy the file", function(done) { + this.Fs.createWriteStream =sinon.stub().returns({ + on(event, handler) { + if (event === 'finish') { return process.nextTick(handler); } + } + }); + return this.FSPersistorManager.sendFile(this.location, this.name1, this.name2, err=> { + this.Fs.createReadStream.calledWith(this.name2).should.equal(true); + this.Fs.createWriteStream.calledWith(`${this.location}/${this.name1Filtered}` ).should.equal(true); + return done(); + }); + }); - it "should return an error if the file cannot be stored", (done) -> - @Fs.createWriteStream =sinon.stub().returns({ - on: (event, handler) => - if event is 'error' - process.nextTick () => - handler(@error) - }) - @FSPersistorManager.sendFile @location, @name1, @name2, (err)=> - @Fs.createReadStream.calledWith(@name2).should.equal true - @Fs.createWriteStream.calledWith("#{@location}/#{@name1Filtered}" ).should.equal true - err.should.equal @error - done() + return it("should return an error if the file cannot be stored", function(done) { + this.Fs.createWriteStream =sinon.stub().returns({ + on: (event, handler) => { + if (event === 'error') { + return process.nextTick(() => { + return handler(this.error); + }); + } + } + }); + return this.FSPersistorManager.sendFile(this.location, this.name1, this.name2, err=> { + this.Fs.createReadStream.calledWith(this.name2).should.equal(true); + this.Fs.createWriteStream.calledWith(`${this.location}/${this.name1Filtered}` ).should.equal(true); + err.should.equal(this.error); + return done(); + }); + }); + }); - describe "sendStream", -> - beforeEach -> - @FSPersistorManager.sendFile = sinon.stub().callsArgWith(3) - @LocalFileWriter.writeStream.callsArgWith(2, null, @name1) - @LocalFileWriter.deleteFile.callsArg(1) - @SourceStream = - on:-> + describe("sendStream", function() { + beforeEach(function() { + this.FSPersistorManager.sendFile = sinon.stub().callsArgWith(3); + this.LocalFileWriter.writeStream.callsArgWith(2, null, this.name1); + this.LocalFileWriter.deleteFile.callsArg(1); + return this.SourceStream = + {on() {}}; + }); - it "should sent stream to LocalFileWriter", (done)-> - @FSPersistorManager.sendStream @location, @name1, @SourceStream, => - @LocalFileWriter.writeStream.calledWith(@SourceStream).should.equal true - done() + it("should sent stream to LocalFileWriter", function(done){ + return this.FSPersistorManager.sendStream(this.location, this.name1, this.SourceStream, () => { + this.LocalFileWriter.writeStream.calledWith(this.SourceStream).should.equal(true); + return done(); + }); + }); - it "should return the error from LocalFileWriter", (done)-> - @LocalFileWriter.writeStream.callsArgWith(2, @error) - @FSPersistorManager.sendStream @location, @name1, @SourceStream, (err)=> - err.should.equal @error - done() + it("should return the error from LocalFileWriter", function(done){ + this.LocalFileWriter.writeStream.callsArgWith(2, this.error); + return this.FSPersistorManager.sendStream(this.location, this.name1, this.SourceStream, err=> { + err.should.equal(this.error); + return done(); + }); + }); - it "should send the file to the filestore", (done)-> - @LocalFileWriter.writeStream.callsArgWith(2) - @FSPersistorManager.sendStream @location, @name1, @SourceStream, (err)=> - @FSPersistorManager.sendFile.called.should.equal true - done() + return it("should send the file to the filestore", function(done){ + this.LocalFileWriter.writeStream.callsArgWith(2); + return this.FSPersistorManager.sendStream(this.location, this.name1, this.SourceStream, err=> { + this.FSPersistorManager.sendFile.called.should.equal(true); + return done(); + }); + }); + }); - describe "getFileStream", -> - beforeEach -> - @opts = {} + describe("getFileStream", function() { + beforeEach(function() { + return this.opts = {};}); - it "should use correct file location", (done) -> - @FSPersistorManager.getFileStream @location, @name1, @opts, (err,res) => - @Fs.open.calledWith("#{@location}/#{@name1Filtered}").should.equal true - done() + it("should use correct file location", function(done) { + this.FSPersistorManager.getFileStream(this.location, this.name1, this.opts, (err,res) => {}); + this.Fs.open.calledWith(`${this.location}/${this.name1Filtered}`).should.equal(true); + return done(); + }); - describe "with start and end options", -> + describe("with start and end options", function() { - beforeEach -> - @fd = 2019 - @opts_in = {start: 0, end: 8} - @opts = {start: 0, end: 8, fd: @fd} - @Fs.open.callsArgWith(2, null, @fd) + beforeEach(function() { + this.fd = 2019; + this.opts_in = {start: 0, end: 8}; + this.opts = {start: 0, end: 8, fd: this.fd}; + return this.Fs.open.callsArgWith(2, null, this.fd); + }); - it 'should pass the options to createReadStream', (done) -> - @FSPersistorManager.getFileStream @location, @name1, @opts_in, (err,res)=> - @Fs.createReadStream.calledWith(null, @opts).should.equal true - done() + return it('should pass the options to createReadStream', function(done) { + this.FSPersistorManager.getFileStream(this.location, this.name1, this.opts_in, (err,res)=> {}); + this.Fs.createReadStream.calledWith(null, this.opts).should.equal(true); + return done(); + }); + }); - describe "error conditions", -> + return describe("error conditions", function() { - describe "when the file does not exist", -> + describe("when the file does not exist", function() { - beforeEach -> - @fakeCode = 'ENOENT' - err = new Error() - err.code = @fakeCode - @Fs.open.callsArgWith(2, err, null) + beforeEach(function() { + this.fakeCode = 'ENOENT'; + const err = new Error(); + err.code = this.fakeCode; + return this.Fs.open.callsArgWith(2, err, null); + }); - it "should give a NotFoundError", (done) -> - @FSPersistorManager.getFileStream @location, @name1, @opts, (err,res)=> - expect(res).to.equal null - expect(err).to.not.equal null - expect(err instanceof @Errors.NotFoundError).to.equal true - done() + return it("should give a NotFoundError", function(done) { + return this.FSPersistorManager.getFileStream(this.location, this.name1, this.opts, (err,res)=> { + expect(res).to.equal(null); + expect(err).to.not.equal(null); + expect(err instanceof this.Errors.NotFoundError).to.equal(true); + return done(); + }); + }); + }); - describe "when some other error happens", -> + return describe("when some other error happens", function() { - beforeEach -> - @fakeCode = 'SOMETHINGHORRIBLE' - err = new Error() - err.code = @fakeCode - @Fs.open.callsArgWith(2, err, null) + beforeEach(function() { + this.fakeCode = 'SOMETHINGHORRIBLE'; + const err = new Error(); + err.code = this.fakeCode; + return this.Fs.open.callsArgWith(2, err, null); + }); - it "should give an Error", (done) -> - @FSPersistorManager.getFileStream @location, @name1, @opts, (err,res)=> - expect(res).to.equal null - expect(err).to.not.equal null - expect(err instanceof Error).to.equal true - done() + return it("should give an Error", function(done) { + return this.FSPersistorManager.getFileStream(this.location, this.name1, this.opts, (err,res)=> { + expect(res).to.equal(null); + expect(err).to.not.equal(null); + expect(err instanceof Error).to.equal(true); + return done(); + }); + }); + }); + }); + }); - describe "getFileSize", -> - it "should return the file size", (done) -> - expectedFileSize = 75382 - @Fs.stat.yields(new Error("fs.stat got unexpected arguments")) - @Fs.stat.withArgs("#{@location}/#{@name1Filtered}") - .yields(null, { size: expectedFileSize }) + describe("getFileSize", function() { + it("should return the file size", function(done) { + const expectedFileSize = 75382; + this.Fs.stat.yields(new Error("fs.stat got unexpected arguments")); + this.Fs.stat.withArgs(`${this.location}/${this.name1Filtered}`) + .yields(null, { size: expectedFileSize }); - @FSPersistorManager.getFileSize @location, @name1, (err, fileSize) => - if err? - return done(err) - expect(fileSize).to.equal(expectedFileSize) - done() + return this.FSPersistorManager.getFileSize(this.location, this.name1, (err, fileSize) => { + if (err != null) { + return done(err); + } + expect(fileSize).to.equal(expectedFileSize); + return done(); + }); + }); - it "should throw a NotFoundError if the file does not exist", (done) -> - error = new Error() - error.code = "ENOENT" - @Fs.stat.yields(error) + it("should throw a NotFoundError if the file does not exist", function(done) { + const error = new Error(); + error.code = "ENOENT"; + this.Fs.stat.yields(error); - @FSPersistorManager.getFileSize @location, @name1, (err, fileSize) => - expect(err).to.be.instanceof(@Errors.NotFoundError) - done() + return this.FSPersistorManager.getFileSize(this.location, this.name1, (err, fileSize) => { + expect(err).to.be.instanceof(this.Errors.NotFoundError); + return done(); + }); + }); - it "should rethrow any other error", (done) -> - error = new Error() - @Fs.stat.yields(error) + return it("should rethrow any other error", function(done) { + const error = new Error(); + this.Fs.stat.yields(error); - @FSPersistorManager.getFileSize @location, @name1, (err, fileSize) => - expect(err).to.equal(error) - done() + return this.FSPersistorManager.getFileSize(this.location, this.name1, (err, fileSize) => { + expect(err).to.equal(error); + return done(); + }); + }); + }); - describe "copyFile", -> - beforeEach -> - @ReadStream= - on:-> + describe("copyFile", function() { + beforeEach(function() { + this.ReadStream= { + on() {}, pipe:sinon.stub() - @WriteStream= - on:-> - @Fs.createReadStream.returns(@ReadStream) - @Fs.createWriteStream.returns(@WriteStream) + }; + this.WriteStream= + {on() {}}; + this.Fs.createReadStream.returns(this.ReadStream); + return this.Fs.createWriteStream.returns(this.WriteStream); + }); - it "Should open the source for reading", (done) -> - @FSPersistorManager.copyFile @location, @name1, @name2, -> - @Fs.createReadStream.calledWith("#{@location}/#{@name1Filtered}").should.equal true - done() + it("Should open the source for reading", function(done) { + this.FSPersistorManager.copyFile(this.location, this.name1, this.name2, function() {}); + this.Fs.createReadStream.calledWith(`${this.location}/${this.name1Filtered}`).should.equal(true); + return done(); + }); - it "Should open the target for writing", (done) -> - @FSPersistorManager.copyFile @location, @name1, @name2, -> - @Fs.createWriteStream.calledWith("#{@location}/#{@name2}").should.equal true - done() + it("Should open the target for writing", function(done) { + this.FSPersistorManager.copyFile(this.location, this.name1, this.name2, function() {}); + this.Fs.createWriteStream.calledWith(`${this.location}/${this.name2}`).should.equal(true); + return done(); + }); - it "Should pipe the source to the target", (done) -> - @FSPersistorManager.copyFile @location, @name1, @name2, -> - @ReadStream.pipe.calledWith(@WriteStream).should.equal true - done() + return it("Should pipe the source to the target", function(done) { + this.FSPersistorManager.copyFile(this.location, this.name1, this.name2, function() {}); + this.ReadStream.pipe.calledWith(this.WriteStream).should.equal(true); + return done(); + }); + }); - describe "deleteFile", -> - beforeEach -> - @Fs.unlink.callsArgWith(1,@error) + describe("deleteFile", function() { + beforeEach(function() { + return this.Fs.unlink.callsArgWith(1,this.error); + }); - it "Should call unlink with correct options", (done) -> - @FSPersistorManager.deleteFile @location, @name1, (err) => - @Fs.unlink.calledWith("#{@location}/#{@name1Filtered}").should.equal true - done() + it("Should call unlink with correct options", function(done) { + return this.FSPersistorManager.deleteFile(this.location, this.name1, err => { + this.Fs.unlink.calledWith(`${this.location}/${this.name1Filtered}`).should.equal(true); + return done(); + }); + }); - it "Should propogate the error", (done) -> - @FSPersistorManager.deleteFile @location, @name1, (err) => - err.should.equal @error - done() + return it("Should propogate the error", function(done) { + return this.FSPersistorManager.deleteFile(this.location, this.name1, err => { + err.should.equal(this.error); + return done(); + }); + }); + }); - describe "deleteDirectory", -> - beforeEach -> - @Rimraf.callsArgWith(1,@error) + describe("deleteDirectory", function() { + beforeEach(function() { + return this.Rimraf.callsArgWith(1,this.error); + }); - it "Should call rmdir(rimraf) with correct options", (done) -> - @FSPersistorManager.deleteDirectory @location, @name1, (err) => - @Rimraf.calledWith("#{@location}/#{@name1Filtered}").should.equal true - done() + it("Should call rmdir(rimraf) with correct options", function(done) { + return this.FSPersistorManager.deleteDirectory(this.location, this.name1, err => { + this.Rimraf.calledWith(`${this.location}/${this.name1Filtered}`).should.equal(true); + return done(); + }); + }); - it "Should propogate the error", (done) -> - @FSPersistorManager.deleteDirectory @location, @name1, (err) => - err.should.equal @error - done() + return it("Should propogate the error", function(done) { + return this.FSPersistorManager.deleteDirectory(this.location, this.name1, err => { + err.should.equal(this.error); + return done(); + }); + }); + }); - describe "checkIfFileExists", -> - beforeEach -> - @Fs.exists.callsArgWith(1,true) + describe("checkIfFileExists", function() { + beforeEach(function() { + return this.Fs.exists.callsArgWith(1,true); + }); - it "Should call exists with correct options", (done) -> - @FSPersistorManager.checkIfFileExists @location, @name1, (exists) => - @Fs.exists.calledWith("#{@location}/#{@name1Filtered}").should.equal true - done() + it("Should call exists with correct options", function(done) { + return this.FSPersistorManager.checkIfFileExists(this.location, this.name1, exists => { + this.Fs.exists.calledWith(`${this.location}/${this.name1Filtered}`).should.equal(true); + return done(); + }); + }); - # fs.exists simply returns false on any error, so... - it "should not return an error", (done) -> - @FSPersistorManager.checkIfFileExists @location, @name1, (err,exists) => - expect(err).to.be.null - done() + // fs.exists simply returns false on any error, so... + it("should not return an error", function(done) { + return this.FSPersistorManager.checkIfFileExists(this.location, this.name1, (err,exists) => { + expect(err).to.be.null; + return done(); + }); + }); - it "Should return true for existing files", (done) -> - @Fs.exists.callsArgWith(1,true) - @FSPersistorManager.checkIfFileExists @location, @name1, (err,exists) => - exists.should.be.true - done() + it("Should return true for existing files", function(done) { + this.Fs.exists.callsArgWith(1,true); + return this.FSPersistorManager.checkIfFileExists(this.location, this.name1, (err,exists) => { + exists.should.be.true; + return done(); + }); + }); - it "Should return false for non-existing files", (done) -> - @Fs.exists.callsArgWith(1,false) - @FSPersistorManager.checkIfFileExists @location, @name1, (err,exists) => - exists.should.be.false - done() + return it("Should return false for non-existing files", function(done) { + this.Fs.exists.callsArgWith(1,false); + return this.FSPersistorManager.checkIfFileExists(this.location, this.name1, (err,exists) => { + exists.should.be.false; + return done(); + }); + }); + }); - describe "directorySize", -> + return describe("directorySize", function() { - it "should propogate the error", (done) -> - @Fs.readdir.callsArgWith(1, @error) - @FSPersistorManager.directorySize @location, @name1, (err, totalsize) => - err.should.equal @error - done() + it("should propogate the error", function(done) { + this.Fs.readdir.callsArgWith(1, this.error); + return this.FSPersistorManager.directorySize(this.location, this.name1, (err, totalsize) => { + err.should.equal(this.error); + return done(); + }); + }); - it "should sum directory files size", (done) -> - @Fs.readdir.callsArgWith(1, null, [ {'file1'}, {'file2'} ]) - @Fs.fstatSync.returns({size : 1024}) - @FSPersistorManager.directorySize @location, @name1, (err, totalsize) => - expect(totalsize).to.equal 2048 - done() + return it("should sum directory files size", function(done) { + this.Fs.readdir.callsArgWith(1, null, [ {'file1': 'file1'}, {'file2': 'file2'} ]); + this.Fs.fstatSync.returns({size : 1024}); + return this.FSPersistorManager.directorySize(this.location, this.name1, (err, totalsize) => { + expect(totalsize).to.equal(2048); + return done(); + }); + }); + }); +}); diff --git a/services/filestore/test/unit/coffee/FileControllerTests.js b/services/filestore/test/unit/coffee/FileControllerTests.js index 821aadb68d..ea3b0e32b8 100644 --- a/services/filestore/test/unit/coffee/FileControllerTests.js +++ b/services/filestore/test/unit/coffee/FileControllerTests.js @@ -1,216 +1,283 @@ -assert = require("chai").assert -sinon = require('sinon') -chai = require('chai') -should = chai.should() -expect = chai.expect -modulePath = "../../../app/js/FileController.js" -SandboxedModule = require('sandboxed-module') +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const { + assert +} = require("chai"); +const sinon = require('sinon'); +const chai = require('chai'); +const should = chai.should(); +const { + expect +} = chai; +const modulePath = "../../../app/js/FileController.js"; +const SandboxedModule = require('sandboxed-module'); -describe "FileController", -> +describe("FileController", function() { - beforeEach -> - @PersistorManager = - sendStream: sinon.stub() - copyFile: sinon.stub() + beforeEach(function() { + this.PersistorManager = { + sendStream: sinon.stub(), + copyFile: sinon.stub(), deleteFile:sinon.stub() + }; - @settings = - s3: - buckets: + this.settings = { + s3: { + buckets: { user_files:"user_files" - @FileHandler = - getFile: sinon.stub() - getFileSize: sinon.stub() - deleteFile: sinon.stub() - insertFile: sinon.stub() + } + } + }; + this.FileHandler = { + getFile: sinon.stub(), + getFileSize: sinon.stub(), + deleteFile: sinon.stub(), + insertFile: sinon.stub(), getDirectorySize: sinon.stub() - @LocalFileWriter = {} - @controller = SandboxedModule.require modulePath, requires: - "./LocalFileWriter":@LocalFileWriter - "./FileHandler": @FileHandler - "./PersistorManager":@PersistorManager - "./Errors": @Errors = - NotFoundError: sinon.stub() - "settings-sharelatex": @settings - "metrics-sharelatex": - inc:-> - "logger-sharelatex": - log:-> - err:-> - @project_id = "project_id" - @file_id = "file_id" - @bucket = "user_files" - @key = "#{@project_id}/#{@file_id}" - @req = - key:@key - bucket:@bucket - query:{} - params: - project_id:@project_id - file_id:@file_id + }; + this.LocalFileWriter = {}; + this.controller = SandboxedModule.require(modulePath, { requires: { + "./LocalFileWriter":this.LocalFileWriter, + "./FileHandler": this.FileHandler, + "./PersistorManager":this.PersistorManager, + "./Errors": (this.Errors = + {NotFoundError: sinon.stub()}), + "settings-sharelatex": this.settings, + "metrics-sharelatex": { + inc() {} + }, + "logger-sharelatex": { + log() {}, + err() {} + } + } + } + ); + this.project_id = "project_id"; + this.file_id = "file_id"; + this.bucket = "user_files"; + this.key = `${this.project_id}/${this.file_id}`; + this.req = { + key:this.key, + bucket:this.bucket, + query:{}, + params: { + project_id:this.project_id, + file_id:this.file_id + }, headers: {} - @res = - set: sinon.stub().returnsThis() + }; + this.res = { + set: sinon.stub().returnsThis(), status: sinon.stub().returnsThis() - @fileStream = {} + }; + return this.fileStream = {};}); - describe "getFile", -> + describe("getFile", function() { - it "should pipe the stream", (done)-> - @FileHandler.getFile.callsArgWith(3, null, @fileStream) - @fileStream.pipe = (res)=> - res.should.equal @res - done() - @controller.getFile @req, @res + it("should pipe the stream", function(done){ + this.FileHandler.getFile.callsArgWith(3, null, this.fileStream); + this.fileStream.pipe = res=> { + res.should.equal(this.res); + return done(); + }; + return this.controller.getFile(this.req, this.res); + }); - it "should send a 200 if the cacheWarm param is true", (done)-> - @req.query.cacheWarm = true - @FileHandler.getFile.callsArgWith(3, null, @fileStream) - @res.send = (statusCode)=> - statusCode.should.equal 200 - done() - @controller.getFile @req, @res + it("should send a 200 if the cacheWarm param is true", function(done){ + this.req.query.cacheWarm = true; + this.FileHandler.getFile.callsArgWith(3, null, this.fileStream); + this.res.send = statusCode=> { + statusCode.should.equal(200); + return done(); + }; + return this.controller.getFile(this.req, this.res); + }); - it "should send a 500 if there is a problem", (done)-> - @FileHandler.getFile.callsArgWith(3, "error") - @res.send = (code)=> - code.should.equal 500 - done() - @controller.getFile @req, @res + it("should send a 500 if there is a problem", function(done){ + this.FileHandler.getFile.callsArgWith(3, "error"); + this.res.send = code=> { + code.should.equal(500); + return done(); + }; + return this.controller.getFile(this.req, this.res); + }); - describe "with a 'Range' header set", -> + return describe("with a 'Range' header set", function() { - beforeEach -> - @req.headers.range = 'bytes=0-8' + beforeEach(function() { + return this.req.headers.range = 'bytes=0-8'; + }); - it "should pass 'start' and 'end' options to FileHandler", (done) -> - @FileHandler.getFile.callsArgWith(3, null, @fileStream) - @fileStream.pipe = (res)=> - expect(@FileHandler.getFile.lastCall.args[2].start).to.equal 0 - expect(@FileHandler.getFile.lastCall.args[2].end).to.equal 8 - done() - @controller.getFile @req, @res + return it("should pass 'start' and 'end' options to FileHandler", function(done) { + this.FileHandler.getFile.callsArgWith(3, null, this.fileStream); + this.fileStream.pipe = res=> { + expect(this.FileHandler.getFile.lastCall.args[2].start).to.equal(0); + expect(this.FileHandler.getFile.lastCall.args[2].end).to.equal(8); + return done(); + }; + return this.controller.getFile(this.req, this.res); + }); + }); + }); - describe "getFileHead", -> - it "should return the file size in a Content-Length header", (done) -> - expectedFileSize = 84921 - @FileHandler.getFileSize.yields( + describe("getFileHead", function() { + it("should return the file size in a Content-Length header", function(done) { + const expectedFileSize = 84921; + this.FileHandler.getFileSize.yields( new Error("FileHandler.getFileSize: unexpected arguments") - ) - @FileHandler.getFileSize.withArgs(@bucket, @key).yields(null, expectedFileSize) + ); + this.FileHandler.getFileSize.withArgs(this.bucket, this.key).yields(null, expectedFileSize); - @res.end = () => - expect(@res.status.lastCall.args[0]).to.equal(200) - expect(@res.set.calledWith("Content-Length", expectedFileSize)).to.equal(true) - done() + this.res.end = () => { + expect(this.res.status.lastCall.args[0]).to.equal(200); + expect(this.res.set.calledWith("Content-Length", expectedFileSize)).to.equal(true); + return done(); + }; - @controller.getFileHead(@req, @res) + return this.controller.getFileHead(this.req, this.res); + }); - it "should return a 404 is the file is not found", (done) -> - @FileHandler.getFileSize.yields(new @Errors.NotFoundError()) + it("should return a 404 is the file is not found", function(done) { + this.FileHandler.getFileSize.yields(new this.Errors.NotFoundError()); - @res.end = () => - expect(@res.status.lastCall.args[0]).to.equal(404) - done() + this.res.end = () => { + expect(this.res.status.lastCall.args[0]).to.equal(404); + return done(); + }; - @controller.getFileHead(@req, @res) + return this.controller.getFileHead(this.req, this.res); + }); - it "should return a 500 on internal errors", (done) -> - @FileHandler.getFileSize.yields(new Error()) + return it("should return a 500 on internal errors", function(done) { + this.FileHandler.getFileSize.yields(new Error()); - @res.end = () => - expect(@res.status.lastCall.args[0]).to.equal(500) - done() + this.res.end = () => { + expect(this.res.status.lastCall.args[0]).to.equal(500); + return done(); + }; - @controller.getFileHead(@req, @res) + return this.controller.getFileHead(this.req, this.res); + }); + }); - describe "insertFile", -> + describe("insertFile", () => it("should send bucket name key and res to PersistorManager", function(done){ + this.FileHandler.insertFile.callsArgWith(3); + this.res.send = () => { + this.FileHandler.insertFile.calledWith(this.bucket, this.key, this.req).should.equal(true); + return done(); + }; + return this.controller.insertFile(this.req, this.res); + })); - it "should send bucket name key and res to PersistorManager", (done)-> - @FileHandler.insertFile.callsArgWith(3) - @res.send = => - @FileHandler.insertFile.calledWith(@bucket, @key, @req).should.equal true - done() - @controller.insertFile @req, @res + describe("copyFile", function() { + beforeEach(function() { + this.oldFile_id = "old_file_id"; + this.oldProject_id = "old_project_id"; + return this.req.body = { + source: { + project_id: this.oldProject_id, + file_id: this.oldFile_id + } + }; + }); - describe "copyFile", -> - beforeEach -> - @oldFile_id = "old_file_id" - @oldProject_id = "old_project_id" - @req.body = - source: - project_id: @oldProject_id - file_id: @oldFile_id + it("should send bucket name and both keys to PersistorManager", function(done){ + this.PersistorManager.copyFile.callsArgWith(3); + this.res.send = code=> { + code.should.equal(200); + this.PersistorManager.copyFile.calledWith(this.bucket, `${this.oldProject_id}/${this.oldFile_id}`, this.key).should.equal(true); + return done(); + }; + return this.controller.copyFile(this.req, this.res); + }); - it "should send bucket name and both keys to PersistorManager", (done)-> - @PersistorManager.copyFile.callsArgWith(3) - @res.send = (code)=> - code.should.equal 200 - @PersistorManager.copyFile.calledWith(@bucket, "#{@oldProject_id}/#{@oldFile_id}", @key).should.equal true - done() - @controller.copyFile @req, @res + it("should send a 404 if the original file was not found", function(done) { + this.PersistorManager.copyFile.callsArgWith(3, new this.Errors.NotFoundError()); + this.res.send = code=> { + code.should.equal(404); + return done(); + }; + return this.controller.copyFile(this.req, this.res); + }); - it "should send a 404 if the original file was not found", (done) -> - @PersistorManager.copyFile.callsArgWith(3, new @Errors.NotFoundError()) - @res.send = (code)=> - code.should.equal 404 - done() - @controller.copyFile @req, @res + return it("should send a 500 if there was an error", function(done){ + this.PersistorManager.copyFile.callsArgWith(3, "error"); + this.res.send = code=> { + code.should.equal(500); + return done(); + }; + return this.controller.copyFile(this.req, this.res); + }); + }); - it "should send a 500 if there was an error", (done)-> - @PersistorManager.copyFile.callsArgWith(3, "error") - @res.send = (code)=> - code.should.equal 500 - done() - @controller.copyFile @req, @res + describe("delete file", function() { - describe "delete file", -> + it("should tell the file handler", function(done){ + this.FileHandler.deleteFile.callsArgWith(2); + this.res.send = code=> { + code.should.equal(204); + this.FileHandler.deleteFile.calledWith(this.bucket, this.key).should.equal(true); + return done(); + }; + return this.controller.deleteFile(this.req, this.res); + }); - it "should tell the file handler", (done)-> - @FileHandler.deleteFile.callsArgWith(2) - @res.send = (code)=> - code.should.equal 204 - @FileHandler.deleteFile.calledWith(@bucket, @key).should.equal true - done() - @controller.deleteFile @req, @res + return it("should send a 500 if there was an error", function(done){ + this.FileHandler.deleteFile.callsArgWith(2, "error"); + this.res.send = function(code){ + code.should.equal(500); + return done(); + }; + return this.controller.deleteFile(this.req, this.res); + }); + }); - it "should send a 500 if there was an error", (done)-> - @FileHandler.deleteFile.callsArgWith(2, "error") - @res.send = (code)-> - code.should.equal 500 - done() - @controller.deleteFile @req, @res + describe("_get_range", function() { - describe "_get_range", -> + it("should parse a valid Range header", function(done) { + const result = this.controller._get_range('bytes=0-200'); + expect(result).to.not.equal(null); + expect(result.start).to.equal(0); + expect(result.end).to.equal(200); + return done(); + }); - it "should parse a valid Range header", (done) -> - result = @controller._get_range('bytes=0-200') - expect(result).to.not.equal null - expect(result.start).to.equal 0 - expect(result.end).to.equal 200 - done() + it("should return null for an invalid Range header", function(done) { + const result = this.controller._get_range('wat'); + expect(result).to.equal(null); + return done(); + }); - it "should return null for an invalid Range header", (done) -> - result = @controller._get_range('wat') - expect(result).to.equal null - done() + return it("should return null for any type other than 'bytes'", function(done) { + const result = this.controller._get_range('carrots=0-200'); + expect(result).to.equal(null); + return done(); + }); + }); - it "should return null for any type other than 'bytes'", (done) -> - result = @controller._get_range('carrots=0-200') - expect(result).to.equal null - done() + return describe("directorySize", function() { - describe "directorySize", -> + it("should return total directory size bytes", function(done) { + this.FileHandler.getDirectorySize.callsArgWith(2, null, 1024); + return this.controller.directorySize(this.req, { json:result=> { + expect(result['total bytes']).to.equal(1024); + return done(); + } + } + ); + }); - it "should return total directory size bytes", (done) -> - @FileHandler.getDirectorySize.callsArgWith(2, null, 1024) - @controller.directorySize @req, json:(result)=> - expect(result['total bytes']).to.equal 1024 - done() - - it "should send a 500 if there was an error", (done)-> - @FileHandler.getDirectorySize.callsArgWith(2, "error") - @res.send = (code)-> - code.should.equal 500 - done() - @controller.directorySize @req, @res + return it("should send a 500 if there was an error", function(done){ + this.FileHandler.getDirectorySize.callsArgWith(2, "error"); + this.res.send = function(code){ + code.should.equal(500); + return done(); + }; + return this.controller.directorySize(this.req, this.res); + }); + }); +}); diff --git a/services/filestore/test/unit/coffee/FileConverterTests.js b/services/filestore/test/unit/coffee/FileConverterTests.js index ed59d56376..c546b61a15 100644 --- a/services/filestore/test/unit/coffee/FileConverterTests.js +++ b/services/filestore/test/unit/coffee/FileConverterTests.js @@ -1,81 +1,112 @@ -assert = require("chai").assert -sinon = require('sinon') -chai = require('chai') -should = chai.should() -expect = chai.expect -modulePath = "../../../app/js/FileConverter.js" -SandboxedModule = require('sandboxed-module') +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const { + assert +} = require("chai"); +const sinon = require('sinon'); +const chai = require('chai'); +const should = chai.should(); +const { + expect +} = chai; +const modulePath = "../../../app/js/FileConverter.js"; +const SandboxedModule = require('sandboxed-module'); -describe "FileConverter", -> +describe("FileConverter", function() { - beforeEach -> + beforeEach(function() { - @safe_exec = sinon.stub() - @converter = SandboxedModule.require modulePath, requires: - "./SafeExec": @safe_exec - "logger-sharelatex": - log:-> - err:-> - "metrics-sharelatex": - inc:-> - Timer:-> - done:-> - "settings-sharelatex": @Settings = - commands: + this.safe_exec = sinon.stub(); + this.converter = SandboxedModule.require(modulePath, { requires: { + "./SafeExec": this.safe_exec, + "logger-sharelatex": { + log() {}, + err() {} + }, + "metrics-sharelatex": { + inc() {}, + Timer() { + return {done() {}}; + } + }, + "settings-sharelatex": (this.Settings = { + commands: { convertCommandPrefix: [] + } + }) + } + }); - @sourcePath = "/this/path/here.eps" - @format = "png" - @error = "Error" + this.sourcePath = "/this/path/here.eps"; + this.format = "png"; + return this.error = "Error"; + }); - describe "convert", -> + describe("convert", function() { - it "should convert the source to the requested format", (done)-> - @safe_exec.callsArgWith(2) - @converter.convert @sourcePath, @format, (err)=> - args = @safe_exec.args[0][0] - args.indexOf("#{@sourcePath}[0]").should.not.equal -1 - args.indexOf("#{@sourcePath}.#{@format}").should.not.equal -1 - done() + it("should convert the source to the requested format", function(done){ + this.safe_exec.callsArgWith(2); + return this.converter.convert(this.sourcePath, this.format, err=> { + const args = this.safe_exec.args[0][0]; + args.indexOf(`${this.sourcePath}[0]`).should.not.equal(-1); + args.indexOf(`${this.sourcePath}.${this.format}`).should.not.equal(-1); + return done(); + }); + }); - it "should return the dest path", (done)-> - @safe_exec.callsArgWith(2) - @converter.convert @sourcePath, @format, (err, destPath)=> - destPath.should.equal "#{@sourcePath}.#{@format}" - done() + it("should return the dest path", function(done){ + this.safe_exec.callsArgWith(2); + return this.converter.convert(this.sourcePath, this.format, (err, destPath)=> { + destPath.should.equal(`${this.sourcePath}.${this.format}`); + return done(); + }); + }); - it "should return the error from convert", (done)-> - @safe_exec.callsArgWith(2, @error) - @converter.convert @sourcePath, @format, (err)=> - err.should.equal @error - done() + it("should return the error from convert", function(done){ + this.safe_exec.callsArgWith(2, this.error); + return this.converter.convert(this.sourcePath, this.format, err=> { + err.should.equal(this.error); + return done(); + }); + }); - it "should not accapt an non aproved format", (done)-> - @safe_exec.callsArgWith(2) - @converter.convert @sourcePath, "ahhhhh", (err)=> - expect(err).to.exist - done() + it("should not accapt an non aproved format", function(done){ + this.safe_exec.callsArgWith(2); + return this.converter.convert(this.sourcePath, "ahhhhh", err=> { + expect(err).to.exist; + return done(); + }); + }); - it "should prefix the command with Settings.commands.convertCommandPrefix", (done) -> - @safe_exec.callsArgWith(2) - @Settings.commands.convertCommandPrefix = ["nice"] - @converter.convert @sourcePath, @format, (err)=> - command = @safe_exec.args[0][0] - command[0].should.equal "nice" - done() + return it("should prefix the command with Settings.commands.convertCommandPrefix", function(done) { + this.safe_exec.callsArgWith(2); + this.Settings.commands.convertCommandPrefix = ["nice"]; + return this.converter.convert(this.sourcePath, this.format, err=> { + const command = this.safe_exec.args[0][0]; + command[0].should.equal("nice"); + return done(); + }); + }); + }); - describe "thumbnail", -> - it "should call converter resize with args", (done)-> - @safe_exec.callsArgWith(2) - @converter.thumbnail @sourcePath, (err)=> - args = @safe_exec.args[0][0] - args.indexOf("#{@sourcePath}[0]").should.not.equal -1 - done() + describe("thumbnail", () => it("should call converter resize with args", function(done){ + this.safe_exec.callsArgWith(2); + return this.converter.thumbnail(this.sourcePath, err=> { + const args = this.safe_exec.args[0][0]; + args.indexOf(`${this.sourcePath}[0]`).should.not.equal(-1); + return done(); + }); + })); - describe "preview", -> - it "should call converter resize with args", (done)-> - @safe_exec.callsArgWith(2) - @converter.preview @sourcePath, (err)=> - args = @safe_exec.args[0][0] - args.indexOf("#{@sourcePath}[0]").should.not.equal -1 - done() + return describe("preview", () => it("should call converter resize with args", function(done){ + this.safe_exec.callsArgWith(2); + return this.converter.preview(this.sourcePath, err=> { + const args = this.safe_exec.args[0][0]; + args.indexOf(`${this.sourcePath}[0]`).should.not.equal(-1); + return done(); + }); + })); +}); diff --git a/services/filestore/test/unit/coffee/FileHandlerTests.js b/services/filestore/test/unit/coffee/FileHandlerTests.js index 754366195e..f83561166f 100644 --- a/services/filestore/test/unit/coffee/FileHandlerTests.js +++ b/services/filestore/test/unit/coffee/FileHandlerTests.js @@ -1,208 +1,275 @@ -assert = require("chai").assert -sinon = require('sinon') -chai = require('chai') -should = chai.should() -expect = chai.expect -modulePath = "../../../app/js/FileHandler.js" -SandboxedModule = require('sandboxed-module') +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const { + assert +} = require("chai"); +const sinon = require('sinon'); +const chai = require('chai'); +const should = chai.should(); +const { + expect +} = chai; +const modulePath = "../../../app/js/FileHandler.js"; +const SandboxedModule = require('sandboxed-module'); -describe "FileHandler", -> +describe("FileHandler", function() { - beforeEach -> - @settings = - s3: - buckets: + beforeEach(function() { + this.settings = { + s3: { + buckets: { user_files:"user_files" - @PersistorManager = - getFileStream: sinon.stub() - checkIfFileExists: sinon.stub() - deleteFile: sinon.stub() - deleteDirectory: sinon.stub() - sendStream: sinon.stub() - insertFile: sinon.stub() - directorySize: sinon.stub() - @LocalFileWriter = - writeStream: sinon.stub() - getStream: sinon.stub() - deleteFile: sinon.stub() - @FileConverter = - convert: sinon.stub() - thumbnail: sinon.stub() - preview: sinon.stub() - @keyBuilder = - addCachingToKey: sinon.stub() - getConvertedFolderKey: sinon.stub() - @ImageOptimiser = - compressPng: sinon.stub() - @handler = SandboxedModule.require modulePath, requires: - "settings-sharelatex": @settings - "./PersistorManager":@PersistorManager - "./LocalFileWriter":@LocalFileWriter - "./FileConverter":@FileConverter - "./KeyBuilder": @keyBuilder - "./ImageOptimiser":@ImageOptimiser - "logger-sharelatex": - log:-> - err:-> - @bucket = "my_bucket" - @key = "key/here" - @stubbedPath = "/var/somewhere/path" - @format = "png" - @formattedStubbedPath = "#{@stubbedPath}.#{@format}" - - describe "insertFile", -> - beforeEach -> - @stream = {} - @PersistorManager.deleteDirectory.callsArgWith(2) - @PersistorManager.sendStream.callsArgWith(3) - - it "should send file to the filestore", (done)-> - @handler.insertFile @bucket, @key, @stream, => - @PersistorManager.sendStream.calledWith(@bucket, @key, @stream).should.equal true - done() - - it "should delete the convetedKey folder", (done)-> - @keyBuilder.getConvertedFolderKey.returns(@stubbedConvetedKey) - @handler.insertFile @bucket, @key, @stream, => - @PersistorManager.deleteDirectory.calledWith(@bucket, @stubbedConvetedKey).should.equal true - done() - - describe "deleteFile", -> - beforeEach -> - @keyBuilder.getConvertedFolderKey.returns(@stubbedConvetedKey) - @PersistorManager.deleteFile.callsArgWith(2) - @PersistorManager.deleteDirectory.callsArgWith(2) - - it "should tell the filestore manager to delete the file", (done)-> - @handler.deleteFile @bucket, @key, => - @PersistorManager.deleteFile.calledWith(@bucket, @key).should.equal true - done() - - it "should tell the filestore manager to delete the cached foler", (done)-> - @handler.deleteFile @bucket, @key, => - @PersistorManager.deleteDirectory.calledWith(@bucket, @stubbedConvetedKey).should.equal true - done() - - describe "getFile", -> - beforeEach -> - @handler._getStandardFile = sinon.stub().callsArgWith(3) - @handler._getConvertedFile = sinon.stub().callsArgWith(3) - - it "should call _getStandardFile if no format or style are defined", (done)-> - - @handler.getFile @bucket, @key, null, => - @handler._getStandardFile.called.should.equal true - @handler._getConvertedFile.called.should.equal false - done() - - it "should pass options to _getStandardFile", (done) -> - options = {start: 0, end: 8} - @handler.getFile @bucket, @key, options, => - expect(@handler._getStandardFile.lastCall.args[2].start).to.equal 0 - expect(@handler._getStandardFile.lastCall.args[2].end).to.equal 8 - done() - - it "should call _getConvertedFile if a format is defined", (done)-> - @handler.getFile @bucket, @key, format:"png", => - @handler._getStandardFile.called.should.equal false - @handler._getConvertedFile.called.should.equal true - done() - - describe "_getStandardFile", -> - - beforeEach -> - @fileStream = {on:->} - @PersistorManager.getFileStream.callsArgWith(3, "err", @fileStream) - - it "should get the stream", (done)-> - @handler.getFile @bucket, @key, null, => - @PersistorManager.getFileStream.calledWith(@bucket, @key).should.equal true - done() - - it "should return the stream and error", (done)-> - @handler.getFile @bucket, @key, null, (err, stream)=> - err.should.equal "err" - stream.should.equal @fileStream - done() - - it "should pass options to PersistorManager", (done) -> - @handler.getFile @bucket, @key, {start: 0, end: 8}, => - expect(@PersistorManager.getFileStream.lastCall.args[2].start).to.equal 0 - expect(@PersistorManager.getFileStream.lastCall.args[2].end).to.equal 8 - done() - - - describe "_getConvertedFile", -> - - it "should getFileStream if it does exists", (done)-> - @PersistorManager.checkIfFileExists.callsArgWith(2, null, true) - @PersistorManager.getFileStream.callsArgWith(3) - @handler._getConvertedFile @bucket, @key, {}, => - @PersistorManager.getFileStream.calledWith(@bucket).should.equal true - done() - - it "should call _getConvertedFileAndCache if it does exists", (done)-> - @PersistorManager.checkIfFileExists.callsArgWith(2, null, false) - @handler._getConvertedFileAndCache = sinon.stub().callsArgWith(4) - @handler._getConvertedFile @bucket, @key, {}, => - @handler._getConvertedFileAndCache.calledWith(@bucket, @key).should.equal true - done() - - describe "_getConvertedFileAndCache", -> - - it "should _convertFile ", (done)-> - @stubbedStream = {"something":"here"} - @localStream = { - on: -> + } } - @PersistorManager.sendFile = sinon.stub().callsArgWith(3) - @LocalFileWriter.getStream = sinon.stub().callsArgWith(1, null, @localStream) - @convetedKey = @key+"converted" - @handler._convertFile = sinon.stub().callsArgWith(3, null, @stubbedPath) - @ImageOptimiser.compressPng = sinon.stub().callsArgWith(1) - @handler._getConvertedFileAndCache @bucket, @key, @convetedKey, {}, (err, fsStream)=> - @handler._convertFile.called.should.equal true - @PersistorManager.sendFile.calledWith(@bucket, @convetedKey, @stubbedPath).should.equal true - @ImageOptimiser.compressPng.calledWith(@stubbedPath).should.equal true - @LocalFileWriter.getStream.calledWith(@stubbedPath).should.equal true - fsStream.should.equal @localStream - done() + }; + this.PersistorManager = { + getFileStream: sinon.stub(), + checkIfFileExists: sinon.stub(), + deleteFile: sinon.stub(), + deleteDirectory: sinon.stub(), + sendStream: sinon.stub(), + insertFile: sinon.stub(), + directorySize: sinon.stub() + }; + this.LocalFileWriter = { + writeStream: sinon.stub(), + getStream: sinon.stub(), + deleteFile: sinon.stub() + }; + this.FileConverter = { + convert: sinon.stub(), + thumbnail: sinon.stub(), + preview: sinon.stub() + }; + this.keyBuilder = { + addCachingToKey: sinon.stub(), + getConvertedFolderKey: sinon.stub() + }; + this.ImageOptimiser = + {compressPng: sinon.stub()}; + this.handler = SandboxedModule.require(modulePath, { requires: { + "settings-sharelatex": this.settings, + "./PersistorManager":this.PersistorManager, + "./LocalFileWriter":this.LocalFileWriter, + "./FileConverter":this.FileConverter, + "./KeyBuilder": this.keyBuilder, + "./ImageOptimiser":this.ImageOptimiser, + "logger-sharelatex": { + log() {}, + err() {} + } + } + } + ); + this.bucket = "my_bucket"; + this.key = "key/here"; + this.stubbedPath = "/var/somewhere/path"; + this.format = "png"; + return this.formattedStubbedPath = `${this.stubbedPath}.${this.format}`; + }); - describe "_convertFile", -> - beforeEach -> - @FileConverter.convert.callsArgWith(2, null, @formattedStubbedPath) - @FileConverter.thumbnail.callsArgWith(1, null, @formattedStubbedPath) - @FileConverter.preview.callsArgWith(1, null, @formattedStubbedPath) - @handler._writeS3FileToDisk = sinon.stub().callsArgWith(3, null, @stubbedPath) - @LocalFileWriter.deleteFile.callsArgWith(1) + describe("insertFile", function() { + beforeEach(function() { + this.stream = {}; + this.PersistorManager.deleteDirectory.callsArgWith(2); + return this.PersistorManager.sendStream.callsArgWith(3); + }); - it "should call thumbnail on the writer path if style was thumbnail was specified", (done)-> - @handler._convertFile @bucket, @key, style:"thumbnail", (err, path)=> - path.should.equal @formattedStubbedPath - @FileConverter.thumbnail.calledWith(@stubbedPath).should.equal true - @LocalFileWriter.deleteFile.calledWith(@stubbedPath).should.equal true - done() + it("should send file to the filestore", function(done){ + return this.handler.insertFile(this.bucket, this.key, this.stream, () => { + this.PersistorManager.sendStream.calledWith(this.bucket, this.key, this.stream).should.equal(true); + return done(); + }); + }); - it "should call preview on the writer path if style was preview was specified", (done)-> - @handler._convertFile @bucket, @key, style:"preview", (err, path)=> - path.should.equal @formattedStubbedPath - @FileConverter.preview.calledWith(@stubbedPath).should.equal true - @LocalFileWriter.deleteFile.calledWith(@stubbedPath).should.equal true - done() + return it("should delete the convetedKey folder", function(done){ + this.keyBuilder.getConvertedFolderKey.returns(this.stubbedConvetedKey); + return this.handler.insertFile(this.bucket, this.key, this.stream, () => { + this.PersistorManager.deleteDirectory.calledWith(this.bucket, this.stubbedConvetedKey).should.equal(true); + return done(); + }); + }); + }); - it "should call convert on the writer path if a format was specified", (done)-> - @handler._convertFile @bucket, @key, format:@format, (err, path)=> - path.should.equal @formattedStubbedPath - @FileConverter.convert.calledWith(@stubbedPath, @format).should.equal true - @LocalFileWriter.deleteFile.calledWith(@stubbedPath).should.equal true - done() + describe("deleteFile", function() { + beforeEach(function() { + this.keyBuilder.getConvertedFolderKey.returns(this.stubbedConvetedKey); + this.PersistorManager.deleteFile.callsArgWith(2); + return this.PersistorManager.deleteDirectory.callsArgWith(2); + }); - describe "getDirectorySize", -> + it("should tell the filestore manager to delete the file", function(done){ + return this.handler.deleteFile(this.bucket, this.key, () => { + this.PersistorManager.deleteFile.calledWith(this.bucket, this.key).should.equal(true); + return done(); + }); + }); - beforeEach -> - @PersistorManager.directorySize.callsArgWith(2) + return it("should tell the filestore manager to delete the cached foler", function(done){ + return this.handler.deleteFile(this.bucket, this.key, () => { + this.PersistorManager.deleteDirectory.calledWith(this.bucket, this.stubbedConvetedKey).should.equal(true); + return done(); + }); + }); + }); - it "should call the filestore manager to get directory size", (done)-> - @handler.getDirectorySize @bucket, @key, => - @PersistorManager.directorySize.calledWith(@bucket, @key).should.equal true - done() + describe("getFile", function() { + beforeEach(function() { + this.handler._getStandardFile = sinon.stub().callsArgWith(3); + return this.handler._getConvertedFile = sinon.stub().callsArgWith(3); + }); + + it("should call _getStandardFile if no format or style are defined", function(done){ + + return this.handler.getFile(this.bucket, this.key, null, () => { + this.handler._getStandardFile.called.should.equal(true); + this.handler._getConvertedFile.called.should.equal(false); + return done(); + }); + }); + + it("should pass options to _getStandardFile", function(done) { + const options = {start: 0, end: 8}; + return this.handler.getFile(this.bucket, this.key, options, () => { + expect(this.handler._getStandardFile.lastCall.args[2].start).to.equal(0); + expect(this.handler._getStandardFile.lastCall.args[2].end).to.equal(8); + return done(); + }); + }); + + return it("should call _getConvertedFile if a format is defined", function(done){ + return this.handler.getFile(this.bucket, this.key, {format:"png"}, () => { + this.handler._getStandardFile.called.should.equal(false); + this.handler._getConvertedFile.called.should.equal(true); + return done(); + }); + }); + }); + + describe("_getStandardFile", function() { + + beforeEach(function() { + this.fileStream = {on() {}}; + return this.PersistorManager.getFileStream.callsArgWith(3, "err", this.fileStream); + }); + + it("should get the stream", function(done){ + return this.handler.getFile(this.bucket, this.key, null, () => { + this.PersistorManager.getFileStream.calledWith(this.bucket, this.key).should.equal(true); + return done(); + }); + }); + + it("should return the stream and error", function(done){ + return this.handler.getFile(this.bucket, this.key, null, (err, stream)=> { + err.should.equal("err"); + stream.should.equal(this.fileStream); + return done(); + }); + }); + + return it("should pass options to PersistorManager", function(done) { + return this.handler.getFile(this.bucket, this.key, {start: 0, end: 8}, () => { + expect(this.PersistorManager.getFileStream.lastCall.args[2].start).to.equal(0); + expect(this.PersistorManager.getFileStream.lastCall.args[2].end).to.equal(8); + return done(); + }); + }); + }); + + + describe("_getConvertedFile", function() { + + it("should getFileStream if it does exists", function(done){ + this.PersistorManager.checkIfFileExists.callsArgWith(2, null, true); + this.PersistorManager.getFileStream.callsArgWith(3); + return this.handler._getConvertedFile(this.bucket, this.key, {}, () => { + this.PersistorManager.getFileStream.calledWith(this.bucket).should.equal(true); + return done(); + }); + }); + + return it("should call _getConvertedFileAndCache if it does exists", function(done){ + this.PersistorManager.checkIfFileExists.callsArgWith(2, null, false); + this.handler._getConvertedFileAndCache = sinon.stub().callsArgWith(4); + return this.handler._getConvertedFile(this.bucket, this.key, {}, () => { + this.handler._getConvertedFileAndCache.calledWith(this.bucket, this.key).should.equal(true); + return done(); + }); + }); + }); + + describe("_getConvertedFileAndCache", () => it("should _convertFile ", function(done){ + this.stubbedStream = {"something":"here"}; + this.localStream = { + on() {} + }; + this.PersistorManager.sendFile = sinon.stub().callsArgWith(3); + this.LocalFileWriter.getStream = sinon.stub().callsArgWith(1, null, this.localStream); + this.convetedKey = this.key+"converted"; + this.handler._convertFile = sinon.stub().callsArgWith(3, null, this.stubbedPath); + this.ImageOptimiser.compressPng = sinon.stub().callsArgWith(1); + return this.handler._getConvertedFileAndCache(this.bucket, this.key, this.convetedKey, {}, (err, fsStream)=> { + this.handler._convertFile.called.should.equal(true); + this.PersistorManager.sendFile.calledWith(this.bucket, this.convetedKey, this.stubbedPath).should.equal(true); + this.ImageOptimiser.compressPng.calledWith(this.stubbedPath).should.equal(true); + this.LocalFileWriter.getStream.calledWith(this.stubbedPath).should.equal(true); + fsStream.should.equal(this.localStream); + return done(); + }); + })); + + describe("_convertFile", function() { + beforeEach(function() { + this.FileConverter.convert.callsArgWith(2, null, this.formattedStubbedPath); + this.FileConverter.thumbnail.callsArgWith(1, null, this.formattedStubbedPath); + this.FileConverter.preview.callsArgWith(1, null, this.formattedStubbedPath); + this.handler._writeS3FileToDisk = sinon.stub().callsArgWith(3, null, this.stubbedPath); + return this.LocalFileWriter.deleteFile.callsArgWith(1); + }); + + it("should call thumbnail on the writer path if style was thumbnail was specified", function(done){ + return this.handler._convertFile(this.bucket, this.key, {style:"thumbnail"}, (err, path)=> { + path.should.equal(this.formattedStubbedPath); + this.FileConverter.thumbnail.calledWith(this.stubbedPath).should.equal(true); + this.LocalFileWriter.deleteFile.calledWith(this.stubbedPath).should.equal(true); + return done(); + }); + }); + + it("should call preview on the writer path if style was preview was specified", function(done){ + return this.handler._convertFile(this.bucket, this.key, {style:"preview"}, (err, path)=> { + path.should.equal(this.formattedStubbedPath); + this.FileConverter.preview.calledWith(this.stubbedPath).should.equal(true); + this.LocalFileWriter.deleteFile.calledWith(this.stubbedPath).should.equal(true); + return done(); + }); + }); + + return it("should call convert on the writer path if a format was specified", function(done){ + return this.handler._convertFile(this.bucket, this.key, {format:this.format}, (err, path)=> { + path.should.equal(this.formattedStubbedPath); + this.FileConverter.convert.calledWith(this.stubbedPath, this.format).should.equal(true); + this.LocalFileWriter.deleteFile.calledWith(this.stubbedPath).should.equal(true); + return done(); + }); + }); + }); + + return describe("getDirectorySize", function() { + + beforeEach(function() { + return this.PersistorManager.directorySize.callsArgWith(2); + }); + + return it("should call the filestore manager to get directory size", function(done){ + return this.handler.getDirectorySize(this.bucket, this.key, () => { + this.PersistorManager.directorySize.calledWith(this.bucket, this.key).should.equal(true); + return done(); + }); + }); + }); +}); diff --git a/services/filestore/test/unit/coffee/ImageOptimiserTests.js b/services/filestore/test/unit/coffee/ImageOptimiserTests.js index 398eacf70f..6074120a56 100644 --- a/services/filestore/test/unit/coffee/ImageOptimiserTests.js +++ b/services/filestore/test/unit/coffee/ImageOptimiserTests.js @@ -1,64 +1,84 @@ -assert = require("chai").assert -sinon = require('sinon') -chai = require('chai') -should = chai.should() -expect = chai.expect -modulePath = "../../../app/js/ImageOptimiser.js" -SandboxedModule = require('sandboxed-module') +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const { + assert +} = require("chai"); +const sinon = require('sinon'); +const chai = require('chai'); +const should = chai.should(); +const { + expect +} = chai; +const modulePath = "../../../app/js/ImageOptimiser.js"; +const SandboxedModule = require('sandboxed-module'); -describe "ImageOptimiser", -> +describe("ImageOptimiser", function() { - beforeEach -> - @child_process = - exec : sinon.stub() - @settings = - enableConversions:true - @optimiser = SandboxedModule.require modulePath, requires: - 'child_process': @child_process - "logger-sharelatex": - log:-> - err:-> - warn:-> - "settings-sharelatex": @settings + beforeEach(function() { + this.child_process = + {exec : sinon.stub()}; + this.settings = + {enableConversions:true}; + this.optimiser = SandboxedModule.require(modulePath, { requires: { + 'child_process': this.child_process, + "logger-sharelatex": { + log() {}, + err() {}, + warn() {} + }, + "settings-sharelatex": this.settings + } + } + ); - @sourcePath = "/this/path/here.eps" - @error = "Error" + this.sourcePath = "/this/path/here.eps"; + return this.error = "Error"; + }); - describe "compressPng", -> + describe("compressPng", function() { - it "convert the file", (done)-> - @child_process.exec.callsArgWith(2) - @optimiser.compressPng @sourcePath, (err)=> - args = @child_process.exec.args[0][0] - args.should.equal "optipng #{@sourcePath}" - done() + it("convert the file", function(done){ + this.child_process.exec.callsArgWith(2); + return this.optimiser.compressPng(this.sourcePath, err=> { + const args = this.child_process.exec.args[0][0]; + args.should.equal(`optipng ${this.sourcePath}`); + return done(); + }); + }); - it "should return the error", (done)-> - @child_process.exec.callsArgWith(2, @error) - @optimiser.compressPng @sourcePath, (err)=> - err.should.equal @error - done() + return it("should return the error", function(done){ + this.child_process.exec.callsArgWith(2, this.error); + return this.optimiser.compressPng(this.sourcePath, err=> { + err.should.equal(this.error); + return done(); + }); + }); + }); - describe 'when enableConversions is disabled', -> - - it 'should produce an error', (done) -> - @settings.enableConversions = false - @child_process.exec.callsArgWith(2) - @optimiser.compressPng @sourcePath, (err)=> - @child_process.exec.called.should.equal false - expect(err).to.exist - done() + describe('when enableConversions is disabled', () => it('should produce an error', function(done) { + this.settings.enableConversions = false; + this.child_process.exec.callsArgWith(2); + return this.optimiser.compressPng(this.sourcePath, err=> { + this.child_process.exec.called.should.equal(false); + expect(err).to.exist; + return done(); + }); + })); - describe 'when optimiser is sigkilled', -> - - it 'should not produce an error', (done) -> - @error = new Error('woops') - @error.signal = 'SIGKILL' - @child_process.exec.callsArgWith(2, @error) - @optimiser.compressPng @sourcePath, (err)=> - expect(err).to.equal(null) - done() + return describe('when optimiser is sigkilled', () => it('should not produce an error', function(done) { + this.error = new Error('woops'); + this.error.signal = 'SIGKILL'; + this.child_process.exec.callsArgWith(2, this.error); + return this.optimiser.compressPng(this.sourcePath, err=> { + expect(err).to.equal(null); + return done(); + }); + })); +}); diff --git a/services/filestore/test/unit/coffee/KeybuilderTests.js b/services/filestore/test/unit/coffee/KeybuilderTests.js index 3ab2dd037a..063a278f23 100644 --- a/services/filestore/test/unit/coffee/KeybuilderTests.js +++ b/services/filestore/test/unit/coffee/KeybuilderTests.js @@ -1,39 +1,59 @@ +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ -assert = require("chai").assert -sinon = require('sinon') -chai = require('chai') -should = chai.should() -expect = chai.expect -modulePath = "../../../app/js/KeyBuilder.js" -SandboxedModule = require('sandboxed-module') +const { + assert +} = require("chai"); +const sinon = require('sinon'); +const chai = require('chai'); +const should = chai.should(); +const { + expect +} = chai; +const modulePath = "../../../app/js/KeyBuilder.js"; +const SandboxedModule = require('sandboxed-module'); -describe "LocalFileWriter", -> +describe("LocalFileWriter", function() { - beforeEach -> + beforeEach(function() { - @keyBuilder = SandboxedModule.require modulePath, requires: - "logger-sharelatex": - log:-> - err:-> - @key = "123/456" + this.keyBuilder = SandboxedModule.require(modulePath, { requires: { + "logger-sharelatex": { + log() {}, + err() {} + } + } + } + ); + return this.key = "123/456"; + }); - describe "cachedKey", -> + return describe("cachedKey", function() { - it "should add the fomat on", -> - opts = + it("should add the fomat on", function() { + const opts = + {format: "png"}; + const newKey = this.keyBuilder.addCachingToKey(this.key, opts); + return newKey.should.equal(`${this.key}-converted-cache/format-png`); + }); + + it("should add the style on", function() { + const opts = + {style: "thumbnail"}; + const newKey = this.keyBuilder.addCachingToKey(this.key, opts); + return newKey.should.equal(`${this.key}-converted-cache/style-thumbnail`); + }); + + return it("should add format on first", function() { + const opts = { + style: "thumbnail", format: "png" - newKey = @keyBuilder.addCachingToKey @key, opts - newKey.should.equal "#{@key}-converted-cache/format-png" - - it "should add the style on", -> - opts = - style: "thumbnail" - newKey = @keyBuilder.addCachingToKey @key, opts - newKey.should.equal "#{@key}-converted-cache/style-thumbnail" - - it "should add format on first", -> - opts = - style: "thumbnail" - format: "png" - newKey = @keyBuilder.addCachingToKey @key, opts - newKey.should.equal "#{@key}-converted-cache/format-png-style-thumbnail" + }; + const newKey = this.keyBuilder.addCachingToKey(this.key, opts); + return newKey.should.equal(`${this.key}-converted-cache/format-png-style-thumbnail`); + }); + }); +}); diff --git a/services/filestore/test/unit/coffee/LocalFileWriterTests.js b/services/filestore/test/unit/coffee/LocalFileWriterTests.js index 55cf1f551f..ed3eb2dfa3 100644 --- a/services/filestore/test/unit/coffee/LocalFileWriterTests.js +++ b/services/filestore/test/unit/coffee/LocalFileWriterTests.js @@ -1,83 +1,121 @@ +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ -assert = require("chai").assert -sinon = require('sinon') -chai = require('chai') -should = chai.should() -expect = chai.expect -modulePath = "../../../app/js/LocalFileWriter.js" -SandboxedModule = require('sandboxed-module') +const { + assert +} = require("chai"); +const sinon = require('sinon'); +const chai = require('chai'); +const should = chai.should(); +const { + expect +} = chai; +const modulePath = "../../../app/js/LocalFileWriter.js"; +const SandboxedModule = require('sandboxed-module'); -describe "LocalFileWriter", -> +describe("LocalFileWriter", function() { - beforeEach -> + beforeEach(function() { - @writeStream = - on: (type, cb)-> - if type == "finish" - cb() - @readStream = - on: -> - @fs = - createWriteStream : sinon.stub().returns(@writeStream) - createReadStream: sinon.stub().returns(@readStream) + this.writeStream = { + on(type, cb){ + if (type === "finish") { + return cb(); + } + } + }; + this.readStream = + {on() {}}; + this.fs = { + createWriteStream : sinon.stub().returns(this.writeStream), + createReadStream: sinon.stub().returns(this.readStream), unlink: sinon.stub() - @settings = - path: + }; + this.settings = { + path: { uploadFolder:"somewhere" - @writer = SandboxedModule.require modulePath, requires: - "fs": @fs - "logger-sharelatex": - log:-> - err:-> - "settings-sharelatex":@settings - "metrics-sharelatex": - inc:-> - Timer:-> - done:-> + } + }; + this.writer = SandboxedModule.require(modulePath, { requires: { + "fs": this.fs, + "logger-sharelatex": { + log() {}, + err() {} + }, + "settings-sharelatex":this.settings, + "metrics-sharelatex": { + inc() {}, + Timer() { + return {done() {}}; + } + } + } + } + ); - @stubbedFsPath = "something/uploads/eio2k1j3" + return this.stubbedFsPath = "something/uploads/eio2k1j3"; + }); - describe "writeStrem", -> - beforeEach -> - @writer._getPath = sinon.stub().returns(@stubbedFsPath) + describe("writeStrem", function() { + beforeEach(function() { + return this.writer._getPath = sinon.stub().returns(this.stubbedFsPath); + }); - it "write the stream to ./uploads", (done)-> - stream = - pipe: (dest)=> - dest.should.equal @writeStream - done() - on: -> - @writer.writeStream stream, null, ()=> + it("write the stream to ./uploads", function(done){ + const stream = { + pipe: dest=> { + dest.should.equal(this.writeStream); + return done(); + }, + on() {} + }; + return this.writer.writeStream(stream, null, ()=> {}); + }); - it "should send the path in the callback", (done)-> - stream = - pipe: (dest)=> - on: (type, cb)-> - if type == "end" - cb() - @writer.writeStream stream, null, (err, fsPath)=> - fsPath.should.equal @stubbedFsPath - done() + return it("should send the path in the callback", function(done){ + const stream = { + pipe: dest=> {}, + on(type, cb){ + if (type === "end") { + return cb(); + } + } + }; + return this.writer.writeStream(stream, null, (err, fsPath)=> { + fsPath.should.equal(this.stubbedFsPath); + return done(); + }); + }); + }); - describe "getStream", -> + describe("getStream", function() { - it "should read the stream from the file ", (done)-> - @writer.getStream @stubbedFsPath, (err, stream)=> - @fs.createReadStream.calledWith(@stubbedFsPath).should.equal true - done() + it("should read the stream from the file ", function(done){ + return this.writer.getStream(this.stubbedFsPath, (err, stream)=> { + this.fs.createReadStream.calledWith(this.stubbedFsPath).should.equal(true); + return done(); + }); + }); - it "should send the stream in the callback", (done)-> - @writer.getStream @stubbedFsPath, (err, readStream)=> - readStream.should.equal @readStream - done() + return it("should send the stream in the callback", function(done){ + return this.writer.getStream(this.stubbedFsPath, (err, readStream)=> { + readStream.should.equal(this.readStream); + return done(); + }); + }); + }); - describe "delete file", -> - - it "should unlink the file", (done)-> - error = "my error" - @fs.unlink.callsArgWith(1, error) - @writer.deleteFile @stubbedFsPath, (err)=> - @fs.unlink.calledWith(@stubbedFsPath).should.equal true - err.should.equal error - done() + return describe("delete file", () => it("should unlink the file", function(done){ + const error = "my error"; + this.fs.unlink.callsArgWith(1, error); + return this.writer.deleteFile(this.stubbedFsPath, err=> { + this.fs.unlink.calledWith(this.stubbedFsPath).should.equal(true); + err.should.equal(error); + return done(); + }); + })); +}); diff --git a/services/filestore/test/unit/coffee/PersistorManagerTests.js b/services/filestore/test/unit/coffee/PersistorManagerTests.js index 3c4ca329e1..d5b859448a 100644 --- a/services/filestore/test/unit/coffee/PersistorManagerTests.js +++ b/services/filestore/test/unit/coffee/PersistorManagerTests.js @@ -1,101 +1,131 @@ -logger = require("logger-sharelatex") -assert = require("chai").assert -sinon = require('sinon') -chai = require('chai') -should = chai.should() -expect = chai.expect -modulePath = "../../../app/js/PersistorManager.js" -SandboxedModule = require('sandboxed-module') +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const logger = require("logger-sharelatex"); +const { + assert +} = require("chai"); +const sinon = require('sinon'); +const chai = require('chai'); +const should = chai.should(); +const { + expect +} = chai; +const modulePath = "../../../app/js/PersistorManager.js"; +const SandboxedModule = require('sandboxed-module'); -describe "PersistorManagerTests", -> +describe("PersistorManagerTests", function() { - beforeEach -> - @S3PersistorManager = - getFileStream: sinon.stub() - checkIfFileExists: sinon.stub() - deleteFile: sinon.stub() - deleteDirectory: sinon.stub() - sendStream: sinon.stub() + beforeEach(function() { + return this.S3PersistorManager = { + getFileStream: sinon.stub(), + checkIfFileExists: sinon.stub(), + deleteFile: sinon.stub(), + deleteDirectory: sinon.stub(), + sendStream: sinon.stub(), insertFile: sinon.stub() + }; + }); - describe "test s3 mixin", -> - beforeEach -> - @settings = - filestore: + describe("test s3 mixin", function() { + beforeEach(function() { + this.settings = { + filestore: { backend: "s3" - @requires = - "./S3PersistorManager": @S3PersistorManager - "settings-sharelatex": @settings - "logger-sharelatex": - log:-> - err:-> - @PersistorManager = SandboxedModule.require modulePath, requires: @requires + } + }; + this.requires = { + "./S3PersistorManager": this.S3PersistorManager, + "settings-sharelatex": this.settings, + "logger-sharelatex": { + log() {}, + err() {} + } + }; + return this.PersistorManager = SandboxedModule.require(modulePath, {requires: this.requires}); + }); - it "should load getFileStream", (done) -> - @PersistorManager.should.respondTo("getFileStream") - @PersistorManager.getFileStream() - @S3PersistorManager.getFileStream.calledOnce.should.equal true - done() + it("should load getFileStream", function(done) { + this.PersistorManager.should.respondTo("getFileStream"); + this.PersistorManager.getFileStream(); + this.S3PersistorManager.getFileStream.calledOnce.should.equal(true); + return done(); + }); - it "should load checkIfFileExists", (done) -> - @PersistorManager.checkIfFileExists() - @S3PersistorManager.checkIfFileExists.calledOnce.should.equal true - done() + it("should load checkIfFileExists", function(done) { + this.PersistorManager.checkIfFileExists(); + this.S3PersistorManager.checkIfFileExists.calledOnce.should.equal(true); + return done(); + }); - it "should load deleteFile", (done) -> - @PersistorManager.deleteFile() - @S3PersistorManager.deleteFile.calledOnce.should.equal true - done() + it("should load deleteFile", function(done) { + this.PersistorManager.deleteFile(); + this.S3PersistorManager.deleteFile.calledOnce.should.equal(true); + return done(); + }); - it "should load deleteDirectory", (done) -> - @PersistorManager.deleteDirectory() - @S3PersistorManager.deleteDirectory.calledOnce.should.equal true - done() + it("should load deleteDirectory", function(done) { + this.PersistorManager.deleteDirectory(); + this.S3PersistorManager.deleteDirectory.calledOnce.should.equal(true); + return done(); + }); - it "should load sendStream", (done) -> - @PersistorManager.sendStream() - @S3PersistorManager.sendStream.calledOnce.should.equal true - done() + it("should load sendStream", function(done) { + this.PersistorManager.sendStream(); + this.S3PersistorManager.sendStream.calledOnce.should.equal(true); + return done(); + }); - it "should load insertFile", (done) -> - @PersistorManager.insertFile() - @S3PersistorManager.insertFile.calledOnce.should.equal true - done() + return it("should load insertFile", function(done) { + this.PersistorManager.insertFile(); + this.S3PersistorManager.insertFile.calledOnce.should.equal(true); + return done(); + }); + }); - describe "test unspecified mixins", -> + describe("test unspecified mixins", () => it("should load s3 when no wrapper specified", function(done) { + this.settings = {filestore:{}}; + this.requires = { + "./S3PersistorManager": this.S3PersistorManager, + "settings-sharelatex": this.settings, + "logger-sharelatex": { + log() {}, + err() {} + } + }; + this.PersistorManager = SandboxedModule.require(modulePath, {requires: this.requires}); + this.PersistorManager.should.respondTo("getFileStream"); + this.PersistorManager.getFileStream(); + this.S3PersistorManager.getFileStream.calledOnce.should.equal(true); + return done(); + })); - it "should load s3 when no wrapper specified", (done) -> - @settings = {filestore:{}} - @requires = - "./S3PersistorManager": @S3PersistorManager - "settings-sharelatex": @settings - "logger-sharelatex": - log:-> - err:-> - @PersistorManager = SandboxedModule.require modulePath, requires: @requires - @PersistorManager.should.respondTo("getFileStream") - @PersistorManager.getFileStream() - @S3PersistorManager.getFileStream.calledOnce.should.equal true - done() - - describe "test invalid mixins", -> - it "should not load an invalid wrapper", (done) -> - @settings = - filestore: - backend:"magic" - @requires = - "./S3PersistorManager": @S3PersistorManager - "settings-sharelatex": @settings - "logger-sharelatex": - log:-> - err:-> - @fsWrapper=null - try - @PersistorManager=SandboxedModule.require modulePath, requires: @requires - catch error - assert.equal("Unknown filestore backend: magic",error.message) - assert.isNull(@fsWrapper) - done() + return describe("test invalid mixins", () => it("should not load an invalid wrapper", function(done) { + this.settings = { + filestore: { + backend:"magic" + } + }; + this.requires = { + "./S3PersistorManager": this.S3PersistorManager, + "settings-sharelatex": this.settings, + "logger-sharelatex": { + log() {}, + err() {} + } + }; + this.fsWrapper=null; + try { + this.PersistorManager=SandboxedModule.require(modulePath, {requires: this.requires}); + } catch (error) { + assert.equal("Unknown filestore backend: magic",error.message); + } + assert.isNull(this.fsWrapper); + return done(); + })); +}); diff --git a/services/filestore/test/unit/coffee/S3PersistorManagerTests.js b/services/filestore/test/unit/coffee/S3PersistorManagerTests.js index a5ab5c2932..4396ce9606 100644 --- a/services/filestore/test/unit/coffee/S3PersistorManagerTests.js +++ b/services/filestore/test/unit/coffee/S3PersistorManagerTests.js @@ -1,343 +1,447 @@ -assert = require("chai").assert -sinon = require('sinon') -chai = require('chai') -should = chai.should() -expect = chai.expect -modulePath = "../../../app/js/S3PersistorManager.js" -SandboxedModule = require('sandboxed-module') +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const { + assert +} = require("chai"); +const sinon = require('sinon'); +const chai = require('chai'); +const should = chai.should(); +const { + expect +} = chai; +const modulePath = "../../../app/js/S3PersistorManager.js"; +const SandboxedModule = require('sandboxed-module'); -describe "S3PersistorManagerTests", -> +describe("S3PersistorManagerTests", function() { - beforeEach -> - @settings = - filestore: - backend: "s3" - s3: - secret: "secret" + beforeEach(function() { + this.settings = { + filestore: { + backend: "s3", + s3: { + secret: "secret", key: "this_key" - stores: + }, + stores: { user_files:"sl_user_files" - @knoxClient = - putFile:sinon.stub() - copyFile:sinon.stub() - list: sinon.stub() - deleteMultiple: sinon.stub() - get: sinon.stub() - @knox = - createClient: sinon.stub().returns(@knoxClient) - @s3EventHandlers = {} - @s3Request = - on: sinon.stub().callsFake (event, callback) => - @s3EventHandlers[event] = callback - send: sinon.stub() - @s3Response = - httpResponse: - createUnbufferedStream: sinon.stub() - @s3Client = - copyObject: sinon.stub() - headObject: sinon.stub() - getObject: sinon.stub().returns(@s3Request) - @awsS3 = sinon.stub().returns(@s3Client) - @LocalFileWriter = - writeStream: sinon.stub() - deleteFile: sinon.stub() - @request = sinon.stub() - @requires = - "knox": @knox - "aws-sdk/clients/s3": @awsS3 - "settings-sharelatex": @settings - "./LocalFileWriter":@LocalFileWriter - "logger-sharelatex": - log:-> - err:-> - "request": @request - "./Errors": @Errors = - NotFoundError: sinon.stub() - @key = "my/key" - @bucketName = "my-bucket" - @error = "my errror" - @S3PersistorManager = SandboxedModule.require modulePath, requires: @requires - - describe "getFileStream", -> - describe "success", -> - beforeEach () -> - @expectedStream = { expectedStream: true } - @expectedStream.on = sinon.stub() - @s3Request.send.callsFake () => - @s3EventHandlers.httpHeaders(200, {}, @s3Response, "OK") - @s3Response.httpResponse.createUnbufferedStream.returns(@expectedStream) - - it "returns a stream", (done) -> - @S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) => - if err? - return done(err) - expect(stream).to.equal(@expectedStream) - done() - - it "sets the AWS client up with credentials from settings", (done) -> - @S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) => - if err? - return done(err) - expect(@awsS3.lastCall.args).to.deep.equal([{ - credentials: - accessKeyId: @settings.filestore.s3.key - secretAccessKey: @settings.filestore.s3.secret - }]) - done() - - it "fetches the right key from the right bucket", (done) -> - @S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) => - if err? - return done(err) - expect(@s3Client.getObject.lastCall.args).to.deep.equal([{ - Bucket: @bucketName, - Key: @key - }]) - done() - - it "accepts alternative credentials", (done) -> - accessKeyId = "that_key" - secret = "that_secret" - opts = { - credentials: - auth_key: accessKeyId - auth_secret: secret } - @S3PersistorManager.getFileStream @bucketName, @key, opts, (err, stream) => - if err? - return done(err) - expect(@awsS3.lastCall.args).to.deep.equal([{ - credentials: - accessKeyId: accessKeyId + } + }; + this.knoxClient = { + putFile:sinon.stub(), + copyFile:sinon.stub(), + list: sinon.stub(), + deleteMultiple: sinon.stub(), + get: sinon.stub() + }; + this.knox = + {createClient: sinon.stub().returns(this.knoxClient)}; + this.s3EventHandlers = {}; + this.s3Request = { + on: sinon.stub().callsFake((event, callback) => { + return this.s3EventHandlers[event] = callback; + }), + send: sinon.stub() + }; + this.s3Response = { + httpResponse: { + createUnbufferedStream: sinon.stub() + } + }; + this.s3Client = { + copyObject: sinon.stub(), + headObject: sinon.stub(), + getObject: sinon.stub().returns(this.s3Request) + }; + this.awsS3 = sinon.stub().returns(this.s3Client); + this.LocalFileWriter = { + writeStream: sinon.stub(), + deleteFile: sinon.stub() + }; + this.request = sinon.stub(); + this.requires = { + "knox": this.knox, + "aws-sdk/clients/s3": this.awsS3, + "settings-sharelatex": this.settings, + "./LocalFileWriter":this.LocalFileWriter, + "logger-sharelatex": { + log() {}, + err() {} + }, + "request": this.request, + "./Errors": (this.Errors = + {NotFoundError: sinon.stub()}) + }; + this.key = "my/key"; + this.bucketName = "my-bucket"; + this.error = "my errror"; + return this.S3PersistorManager = SandboxedModule.require(modulePath, {requires: this.requires}); + }); + + describe("getFileStream", function() { + describe("success", function() { + beforeEach(function() { + this.expectedStream = { expectedStream: true }; + this.expectedStream.on = sinon.stub(); + this.s3Request.send.callsFake(() => { + return this.s3EventHandlers.httpHeaders(200, {}, this.s3Response, "OK"); + }); + return this.s3Response.httpResponse.createUnbufferedStream.returns(this.expectedStream); + }); + + it("returns a stream", function(done) { + return this.S3PersistorManager.getFileStream(this.bucketName, this.key, {}, (err, stream) => { + if (err != null) { + return done(err); + } + expect(stream).to.equal(this.expectedStream); + return done(); + }); + }); + + it("sets the AWS client up with credentials from settings", function(done) { + return this.S3PersistorManager.getFileStream(this.bucketName, this.key, {}, (err, stream) => { + if (err != null) { + return done(err); + } + expect(this.awsS3.lastCall.args).to.deep.equal([{ + credentials: { + accessKeyId: this.settings.filestore.s3.key, + secretAccessKey: this.settings.filestore.s3.secret + } + }]); + return done(); + }); + }); + + it("fetches the right key from the right bucket", function(done) { + return this.S3PersistorManager.getFileStream(this.bucketName, this.key, {}, (err, stream) => { + if (err != null) { + return done(err); + } + expect(this.s3Client.getObject.lastCall.args).to.deep.equal([{ + Bucket: this.bucketName, + Key: this.key + }]); + return done(); + }); + }); + + it("accepts alternative credentials", function(done) { + const accessKeyId = "that_key"; + const secret = "that_secret"; + const opts = { + credentials: { + auth_key: accessKeyId, + auth_secret: secret + } + }; + return this.S3PersistorManager.getFileStream(this.bucketName, this.key, opts, (err, stream) => { + if (err != null) { + return done(err); + } + expect(this.awsS3.lastCall.args).to.deep.equal([{ + credentials: { + accessKeyId, secretAccessKey: secret - }]) - expect(stream).to.equal(@expectedStream) - done() + } + }]); + expect(stream).to.equal(this.expectedStream); + return done(); + }); + }); - it "accepts byte range", (done) -> - start = 0 - end = 8 - opts = { start: start, end: end } - @S3PersistorManager.getFileStream @bucketName, @key, opts, (err, stream) => - if err? - return done(err) - expect(@s3Client.getObject.lastCall.args).to.deep.equal([{ - Bucket: @bucketName - Key: @key - Range: "bytes=#{start}-#{end}" - }]) - expect(stream).to.equal(@expectedStream) - done() + return it("accepts byte range", function(done) { + const start = 0; + const end = 8; + const opts = { start, end }; + return this.S3PersistorManager.getFileStream(this.bucketName, this.key, opts, (err, stream) => { + if (err != null) { + return done(err); + } + expect(this.s3Client.getObject.lastCall.args).to.deep.equal([{ + Bucket: this.bucketName, + Key: this.key, + Range: `bytes=${start}-${end}` + }]); + expect(stream).to.equal(this.expectedStream); + return done(); + }); + }); + }); - describe "errors", -> - describe "when the file doesn't exist", -> - beforeEach -> - @s3Request.send.callsFake () => - @s3EventHandlers.httpHeaders(404, {}, @s3Response, "Not found") + return describe("errors", function() { + describe("when the file doesn't exist", function() { + beforeEach(function() { + return this.s3Request.send.callsFake(() => { + return this.s3EventHandlers.httpHeaders(404, {}, this.s3Response, "Not found"); + }); + }); - it "returns a NotFoundError that indicates the bucket and key", (done) -> - @S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) => - expect(err).to.be.instanceof(@Errors.NotFoundError) - errMsg = @Errors.NotFoundError.lastCall.args[0] - expect(errMsg).to.match(new RegExp(".*#{@bucketName}.*")) - expect(errMsg).to.match(new RegExp(".*#{@key}.*")) - done() + return it("returns a NotFoundError that indicates the bucket and key", function(done) { + return this.S3PersistorManager.getFileStream(this.bucketName, this.key, {}, (err, stream) => { + expect(err).to.be.instanceof(this.Errors.NotFoundError); + const errMsg = this.Errors.NotFoundError.lastCall.args[0]; + expect(errMsg).to.match(new RegExp(`.*${this.bucketName}.*`)); + expect(errMsg).to.match(new RegExp(`.*${this.key}.*`)); + return done(); + }); + }); + }); - describe "when S3 encounters an unkown error", -> - beforeEach -> - @s3Request.send.callsFake () => - @s3EventHandlers.httpHeaders(500, {}, @s3Response, "Internal server error") + describe("when S3 encounters an unkown error", function() { + beforeEach(function() { + return this.s3Request.send.callsFake(() => { + return this.s3EventHandlers.httpHeaders(500, {}, this.s3Response, "Internal server error"); + }); + }); - it "returns an error", (done) -> - @S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) => - expect(err).to.be.instanceof(Error) - done() + return it("returns an error", function(done) { + return this.S3PersistorManager.getFileStream(this.bucketName, this.key, {}, (err, stream) => { + expect(err).to.be.instanceof(Error); + return done(); + }); + }); + }); - describe "when the S3 request errors out before receiving HTTP headers", -> - beforeEach -> - @s3Request.send.callsFake () => - @s3EventHandlers.error(new Error("connection failed")) + return describe("when the S3 request errors out before receiving HTTP headers", function() { + beforeEach(function() { + return this.s3Request.send.callsFake(() => { + return this.s3EventHandlers.error(new Error("connection failed")); + }); + }); - it "returns an error", (done) -> - @S3PersistorManager.getFileStream @bucketName, @key, {}, (err, stream) => - expect(err).to.be.instanceof(Error) - done() + return it("returns an error", function(done) { + return this.S3PersistorManager.getFileStream(this.bucketName, this.key, {}, (err, stream) => { + expect(err).to.be.instanceof(Error); + return done(); + }); + }); + }); + }); + }); - describe "getFileSize", -> - it "should obtain the file size from S3", (done) -> - expectedFileSize = 123 - @s3Client.headObject.yields(new Error( + describe("getFileSize", function() { + it("should obtain the file size from S3", function(done) { + const expectedFileSize = 123; + this.s3Client.headObject.yields(new Error( "s3Client.headObject got unexpected arguments" - )) - @s3Client.headObject.withArgs({ - Bucket: @bucketName - Key: @key - }).yields(null, { ContentLength: expectedFileSize }) + )); + this.s3Client.headObject.withArgs({ + Bucket: this.bucketName, + Key: this.key + }).yields(null, { ContentLength: expectedFileSize }); - @S3PersistorManager.getFileSize @bucketName, @key, (err, fileSize) => - if err? - return done(err) - expect(fileSize).to.equal(expectedFileSize) - done() + return this.S3PersistorManager.getFileSize(this.bucketName, this.key, (err, fileSize) => { + if (err != null) { + return done(err); + } + expect(fileSize).to.equal(expectedFileSize); + return done(); + }); + }); - [403, 404].forEach (statusCode) -> - it "should throw NotFoundError when S3 responds with #{statusCode}", (done) -> - error = new Error() - error.statusCode = statusCode - @s3Client.headObject.yields(error) + [403, 404].forEach(statusCode => it(`should throw NotFoundError when S3 responds with ${statusCode}`, function(done) { + const error = new Error(); + error.statusCode = statusCode; + this.s3Client.headObject.yields(error); - @S3PersistorManager.getFileSize @bucketName, @key, (err, fileSize) => - expect(err).to.be.an.instanceof(@Errors.NotFoundError) - done() + return this.S3PersistorManager.getFileSize(this.bucketName, this.key, (err, fileSize) => { + expect(err).to.be.an.instanceof(this.Errors.NotFoundError); + return done(); + }); + })); - it "should rethrow any other error", (done) -> - error = new Error() - @s3Client.headObject.yields(error) - @s3Client.headObject.yields(error) + return it("should rethrow any other error", function(done) { + const error = new Error(); + this.s3Client.headObject.yields(error); + this.s3Client.headObject.yields(error); - @S3PersistorManager.getFileSize @bucketName, @key, (err, fileSize) => - expect(err).to.equal(error) - done() + return this.S3PersistorManager.getFileSize(this.bucketName, this.key, (err, fileSize) => { + expect(err).to.equal(error); + return done(); + }); + }); + }); - describe "sendFile", -> + describe("sendFile", function() { - beforeEach -> - @knoxClient.putFile.returns on:-> + beforeEach(function() { + return this.knoxClient.putFile.returns({on() {}}); + }); - it "should put file with knox", (done)-> - @LocalFileWriter.deleteFile.callsArgWith(1) - @knoxClient.putFile.callsArgWith(2, @error) - @S3PersistorManager.sendFile @bucketName, @key, @fsPath, (err)=> - @knoxClient.putFile.calledWith(@fsPath, @key).should.equal true - err.should.equal @error - done() + it("should put file with knox", function(done){ + this.LocalFileWriter.deleteFile.callsArgWith(1); + this.knoxClient.putFile.callsArgWith(2, this.error); + return this.S3PersistorManager.sendFile(this.bucketName, this.key, this.fsPath, err=> { + this.knoxClient.putFile.calledWith(this.fsPath, this.key).should.equal(true); + err.should.equal(this.error); + return done(); + }); + }); - it "should delete the file and pass the error with it", (done)-> - @LocalFileWriter.deleteFile.callsArgWith(1) - @knoxClient.putFile.callsArgWith(2, @error) - @S3PersistorManager.sendFile @bucketName, @key, @fsPath, (err)=> - @knoxClient.putFile.calledWith(@fsPath, @key).should.equal true - err.should.equal @error - done() + return it("should delete the file and pass the error with it", function(done){ + this.LocalFileWriter.deleteFile.callsArgWith(1); + this.knoxClient.putFile.callsArgWith(2, this.error); + return this.S3PersistorManager.sendFile(this.bucketName, this.key, this.fsPath, err=> { + this.knoxClient.putFile.calledWith(this.fsPath, this.key).should.equal(true); + err.should.equal(this.error); + return done(); + }); + }); + }); - describe "sendStream", -> - beforeEach -> - @fsPath = "to/some/where" - @origin = - on:-> - @S3PersistorManager.sendFile = sinon.stub().callsArgWith(3) + describe("sendStream", function() { + beforeEach(function() { + this.fsPath = "to/some/where"; + this.origin = + {on() {}}; + return this.S3PersistorManager.sendFile = sinon.stub().callsArgWith(3); + }); - it "should send stream to LocalFileWriter", (done)-> - @LocalFileWriter.deleteFile.callsArgWith(1) - @LocalFileWriter.writeStream.callsArgWith(2, null, @fsPath) - @S3PersistorManager.sendStream @bucketName, @key, @origin, => - @LocalFileWriter.writeStream.calledWith(@origin).should.equal true - done() + it("should send stream to LocalFileWriter", function(done){ + this.LocalFileWriter.deleteFile.callsArgWith(1); + this.LocalFileWriter.writeStream.callsArgWith(2, null, this.fsPath); + return this.S3PersistorManager.sendStream(this.bucketName, this.key, this.origin, () => { + this.LocalFileWriter.writeStream.calledWith(this.origin).should.equal(true); + return done(); + }); + }); - it "should return the error from LocalFileWriter", (done)-> - @LocalFileWriter.deleteFile.callsArgWith(1) - @LocalFileWriter.writeStream.callsArgWith(2, @error) - @S3PersistorManager.sendStream @bucketName, @key, @origin, (err)=> - err.should.equal @error - done() + it("should return the error from LocalFileWriter", function(done){ + this.LocalFileWriter.deleteFile.callsArgWith(1); + this.LocalFileWriter.writeStream.callsArgWith(2, this.error); + return this.S3PersistorManager.sendStream(this.bucketName, this.key, this.origin, err=> { + err.should.equal(this.error); + return done(); + }); + }); - it "should send the file to the filestore", (done)-> - @LocalFileWriter.deleteFile.callsArgWith(1) - @LocalFileWriter.writeStream.callsArgWith(2) - @S3PersistorManager.sendStream @bucketName, @key, @origin, (err)=> - @S3PersistorManager.sendFile.called.should.equal true - done() + return it("should send the file to the filestore", function(done){ + this.LocalFileWriter.deleteFile.callsArgWith(1); + this.LocalFileWriter.writeStream.callsArgWith(2); + return this.S3PersistorManager.sendStream(this.bucketName, this.key, this.origin, err=> { + this.S3PersistorManager.sendFile.called.should.equal(true); + return done(); + }); + }); + }); - describe "copyFile", -> - beforeEach -> - @sourceKey = "my/key" - @destKey = "my/dest/key" + describe("copyFile", function() { + beforeEach(function() { + this.sourceKey = "my/key"; + return this.destKey = "my/dest/key"; + }); - it "should use AWS SDK to copy file", (done)-> - @s3Client.copyObject.callsArgWith(1, @error) - @S3PersistorManager.copyFile @bucketName, @sourceKey, @destKey, (err)=> - err.should.equal @error - @s3Client.copyObject.calledWith({Bucket: @bucketName, Key: @destKey, CopySource: @bucketName + '/' + @key}).should.equal true - done() + it("should use AWS SDK to copy file", function(done){ + this.s3Client.copyObject.callsArgWith(1, this.error); + return this.S3PersistorManager.copyFile(this.bucketName, this.sourceKey, this.destKey, err=> { + err.should.equal(this.error); + this.s3Client.copyObject.calledWith({Bucket: this.bucketName, Key: this.destKey, CopySource: this.bucketName + '/' + this.key}).should.equal(true); + return done(); + }); + }); - it "should return a NotFoundError object if the original file does not exist", (done)-> - NoSuchKeyError = {code: "NoSuchKey"} - @s3Client.copyObject.callsArgWith(1, NoSuchKeyError) - @S3PersistorManager.copyFile @bucketName, @sourceKey, @destKey, (err)=> - expect(err instanceof @Errors.NotFoundError).to.equal true - done() + return it("should return a NotFoundError object if the original file does not exist", function(done){ + const NoSuchKeyError = {code: "NoSuchKey"}; + this.s3Client.copyObject.callsArgWith(1, NoSuchKeyError); + return this.S3PersistorManager.copyFile(this.bucketName, this.sourceKey, this.destKey, err=> { + expect(err instanceof this.Errors.NotFoundError).to.equal(true); + return done(); + }); + }); + }); - describe "deleteDirectory", -> + describe("deleteDirectory", () => it("should list the contents passing them onto multi delete", function(done){ + const data = + {Contents: [{Key:"1234"}, {Key: "456"}]}; + this.knoxClient.list.callsArgWith(1, null, data); + this.knoxClient.deleteMultiple.callsArgWith(1); + return this.S3PersistorManager.deleteDirectory(this.bucketName, this.key, err=> { + this.knoxClient.deleteMultiple.calledWith(["1234","456"]).should.equal(true); + return done(); + }); + })); - it "should list the contents passing them onto multi delete", (done)-> - data = - Contents: [{Key:"1234"}, {Key: "456"}] - @knoxClient.list.callsArgWith(1, null, data) - @knoxClient.deleteMultiple.callsArgWith(1) - @S3PersistorManager.deleteDirectory @bucketName, @key, (err)=> - @knoxClient.deleteMultiple.calledWith(["1234","456"]).should.equal true - done() + describe("deleteFile", function() { - describe "deleteFile", -> + it("should use correct options", function(done){ + this.request.callsArgWith(1); - it "should use correct options", (done)-> - @request.callsArgWith(1) + return this.S3PersistorManager.deleteFile(this.bucketName, this.key, err=> { + const opts = this.request.args[0][0]; + assert.deepEqual(opts.aws, {key:this.settings.filestore.s3.key, secret:this.settings.filestore.s3.secret, bucket:this.bucketName}); + opts.method.should.equal("delete"); + opts.timeout.should.equal((30*1000)); + opts.uri.should.equal(`https://${this.bucketName}.s3.amazonaws.com/${this.key}`); + return done(); + }); + }); - @S3PersistorManager.deleteFile @bucketName, @key, (err)=> - opts = @request.args[0][0] - assert.deepEqual(opts.aws, {key:@settings.filestore.s3.key, secret:@settings.filestore.s3.secret, bucket:@bucketName}) - opts.method.should.equal "delete" - opts.timeout.should.equal (30*1000) - opts.uri.should.equal "https://#{@bucketName}.s3.amazonaws.com/#{@key}" - done() + return it("should return the error", function(done){ + this.request.callsArgWith(1, this.error); - it "should return the error", (done)-> - @request.callsArgWith(1, @error) + return this.S3PersistorManager.deleteFile(this.bucketName, this.key, err=> { + err.should.equal(this.error); + return done(); + }); + }); + }); - @S3PersistorManager.deleteFile @bucketName, @key, (err)=> - err.should.equal @error - done() + describe("checkIfFileExists", function() { - describe "checkIfFileExists", -> + it("should use correct options", function(done){ + this.request.callsArgWith(1, null, {statusCode:200}); - it "should use correct options", (done)-> - @request.callsArgWith(1, null, statusCode:200) + return this.S3PersistorManager.checkIfFileExists(this.bucketName, this.key, err=> { + const opts = this.request.args[0][0]; + assert.deepEqual(opts.aws, {key:this.settings.filestore.s3.key, secret:this.settings.filestore.s3.secret, bucket:this.bucketName}); + opts.method.should.equal("head"); + opts.timeout.should.equal((30*1000)); + opts.uri.should.equal(`https://${this.bucketName}.s3.amazonaws.com/${this.key}`); + return done(); + }); + }); - @S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=> - opts = @request.args[0][0] - assert.deepEqual(opts.aws, {key:@settings.filestore.s3.key, secret:@settings.filestore.s3.secret, bucket:@bucketName}) - opts.method.should.equal "head" - opts.timeout.should.equal (30*1000) - opts.uri.should.equal "https://#{@bucketName}.s3.amazonaws.com/#{@key}" - done() + it("should return true for a 200", function(done){ + this.request.callsArgWith(1, null, {statusCode:200}); - it "should return true for a 200", (done)-> - @request.callsArgWith(1, null, statusCode:200) + return this.S3PersistorManager.checkIfFileExists(this.bucketName, this.key, (err, exists)=> { + exists.should.equal(true); + return done(); + }); + }); - @S3PersistorManager.checkIfFileExists @bucketName, @key, (err, exists)=> - exists.should.equal true - done() + it("should return false for a non 200", function(done){ + this.request.callsArgWith(1, null, {statusCode:404}); - it "should return false for a non 200", (done)-> - @request.callsArgWith(1, null, statusCode:404) + return this.S3PersistorManager.checkIfFileExists(this.bucketName, this.key, (err, exists)=> { + exists.should.equal(false); + return done(); + }); + }); - @S3PersistorManager.checkIfFileExists @bucketName, @key, (err, exists)=> - exists.should.equal false - done() + return it("should return the error", function(done){ + this.request.callsArgWith(1, this.error, {}); - it "should return the error", (done)-> - @request.callsArgWith(1, @error, {}) + return this.S3PersistorManager.checkIfFileExists(this.bucketName, this.key, err=> { + err.should.equal(this.error); + return done(); + }); + }); + }); - @S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=> - err.should.equal @error - done() - - describe "directorySize", -> - - it "should sum directory files size", (done) -> - data = - Contents: [ {Size: 1024}, {Size: 2048} ] - @knoxClient.list.callsArgWith(1, null, data) - @S3PersistorManager.directorySize @bucketName, @key, (err, totalSize)=> - totalSize.should.equal 3072 - done() + return describe("directorySize", () => it("should sum directory files size", function(done) { + const data = + {Contents: [ {Size: 1024}, {Size: 2048} ]}; + this.knoxClient.list.callsArgWith(1, null, data); + return this.S3PersistorManager.directorySize(this.bucketName, this.key, (err, totalSize)=> { + totalSize.should.equal(3072); + return done(); + }); + })); +}); diff --git a/services/filestore/test/unit/coffee/SafeExecTests.js b/services/filestore/test/unit/coffee/SafeExecTests.js index 1be22f3993..4c462f887d 100644 --- a/services/filestore/test/unit/coffee/SafeExecTests.js +++ b/services/filestore/test/unit/coffee/SafeExecTests.js @@ -1,50 +1,75 @@ -assert = require("chai").assert -sinon = require('sinon') -chai = require('chai') -should = chai.should() -expect = chai.expect -modulePath = "../../../app/js/SafeExec.js" -SandboxedModule = require('sandboxed-module') +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const { + assert +} = require("chai"); +const sinon = require('sinon'); +const chai = require('chai'); +const should = chai.should(); +const { + expect +} = chai; +const modulePath = "../../../app/js/SafeExec.js"; +const SandboxedModule = require('sandboxed-module'); -describe "SafeExec", -> +describe("SafeExec", function() { - beforeEach -> - @settings = - enableConversions:true - @safe_exec = SandboxedModule.require modulePath, requires: - "logger-sharelatex": - log:-> - err:-> - "settings-sharelatex": @settings - @options = {timeout: 10*1000, killSignal: "SIGTERM" } + beforeEach(function() { + this.settings = + {enableConversions:true}; + this.safe_exec = SandboxedModule.require(modulePath, { requires: { + "logger-sharelatex": { + log() {}, + err() {} + }, + "settings-sharelatex": this.settings + } + } + ); + return this.options = {timeout: 10*1000, killSignal: "SIGTERM" };}); - describe "safe_exec", -> + return describe("safe_exec", function() { - it "should execute a valid command", (done) -> - @safe_exec ["/bin/echo", "hello"], @options, (err, stdout, stderr) => - stdout.should.equal "hello\n" - should.not.exist(err) - done() + it("should execute a valid command", function(done) { + return this.safe_exec(["/bin/echo", "hello"], this.options, (err, stdout, stderr) => { + stdout.should.equal("hello\n"); + should.not.exist(err); + return done(); + }); + }); - it "should error when conversions are disabled", (done) -> - @settings.enableConversions = false - @safe_exec ["/bin/echo", "hello"], @options, (err, stdout, stderr) => - expect(err).to.exist - done() + it("should error when conversions are disabled", function(done) { + this.settings.enableConversions = false; + return this.safe_exec(["/bin/echo", "hello"], this.options, (err, stdout, stderr) => { + expect(err).to.exist; + return done(); + }); + }); - it "should execute a command with non-zero exit status", (done) -> - @safe_exec ["/usr/bin/env", "false"], @options, (err, stdout, stderr) => - stdout.should.equal "" - stderr.should.equal "" - err.message.should.equal "exit status 1" - done() + it("should execute a command with non-zero exit status", function(done) { + return this.safe_exec(["/usr/bin/env", "false"], this.options, (err, stdout, stderr) => { + stdout.should.equal(""); + stderr.should.equal(""); + err.message.should.equal("exit status 1"); + return done(); + }); + }); - it "should handle an invalid command", (done) -> - @safe_exec ["/bin/foobar"], @options, (err, stdout, stderr) => - err.code.should.equal "ENOENT" - done() + it("should handle an invalid command", function(done) { + return this.safe_exec(["/bin/foobar"], this.options, (err, stdout, stderr) => { + err.code.should.equal("ENOENT"); + return done(); + }); + }); - it "should handle a command that runs too long", (done) -> - @safe_exec ["/bin/sleep", "10"], {timeout: 500, killSignal: "SIGTERM"}, (err, stdout, stderr) => - err.should.equal "SIGTERM" - done() + return it("should handle a command that runs too long", function(done) { + return this.safe_exec(["/bin/sleep", "10"], {timeout: 500, killSignal: "SIGTERM"}, (err, stdout, stderr) => { + err.should.equal("SIGTERM"); + return done(); + }); + }); + }); +}); diff --git a/services/filestore/test/unit/coffee/SettingsTests.js b/services/filestore/test/unit/coffee/SettingsTests.js index efb690511e..5c1b79d70c 100644 --- a/services/filestore/test/unit/coffee/SettingsTests.js +++ b/services/filestore/test/unit/coffee/SettingsTests.js @@ -1,19 +1,29 @@ -assert = require("chai").assert -sinon = require('sinon') -chai = require('chai') -should = chai.should() -expect = chai.expect -modulePath = "../../../app/js/BucketController.js" +/* + * decaffeinate suggestions: + * DS102: Remove unnecessary code created because of implicit returns + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +const { + assert +} = require("chai"); +const sinon = require('sinon'); +const chai = require('chai'); +const should = chai.should(); +const { + expect +} = chai; +const modulePath = "../../../app/js/BucketController.js"; -describe "Settings", -> - describe "s3", -> - it "should use JSONified env var if present", (done)-> - s3_settings = - bucket1: - auth_key: 'bucket1_key' - auth_secret: 'bucket1_secret' - process.env['S3_BUCKET_CREDENTIALS'] = JSON.stringify s3_settings +describe("Settings", () => describe("s3", () => it("should use JSONified env var if present", function(done){ + const s3_settings = { + bucket1: { + auth_key: 'bucket1_key', + auth_secret: 'bucket1_secret' + } + }; + process.env['S3_BUCKET_CREDENTIALS'] = JSON.stringify(s3_settings); - settings = require("settings-sharelatex") - expect(settings.filestore.s3BucketCreds).to.deep.equal s3_settings - done() + const settings = require("settings-sharelatex"); + expect(settings.filestore.s3BucketCreds).to.deep.equal(s3_settings); + return done(); +}))); From ff04085bddf763fdd4cf050a41d811e5e042fe54 Mon Sep 17 00:00:00 2001 From: decaffeinate Date: Mon, 16 Dec 2019 11:20:25 +0000 Subject: [PATCH 24/73] decaffeinate: Run post-processing cleanups on AWSSDKPersistorManagerTests.coffee and 12 other files --- .../test/unit/coffee/AWSSDKPersistorManagerTests.js | 8 ++++++++ .../filestore/test/unit/coffee/BucketControllerTests.js | 6 ++++++ .../filestore/test/unit/coffee/FSPersistorManagerTests.js | 7 +++++++ .../filestore/test/unit/coffee/FileControllerTests.js | 6 ++++++ services/filestore/test/unit/coffee/FileConverterTests.js | 7 +++++++ services/filestore/test/unit/coffee/FileHandlerTests.js | 7 +++++++ .../filestore/test/unit/coffee/ImageOptimiserTests.js | 7 +++++++ services/filestore/test/unit/coffee/KeybuilderTests.js | 6 ++++++ .../filestore/test/unit/coffee/LocalFileWriterTests.js | 7 +++++++ .../filestore/test/unit/coffee/PersistorManagerTests.js | 6 ++++++ .../filestore/test/unit/coffee/S3PersistorManagerTests.js | 7 +++++++ services/filestore/test/unit/coffee/SafeExecTests.js | 6 ++++++ services/filestore/test/unit/coffee/SettingsTests.js | 8 +++++++- 13 files changed, 87 insertions(+), 1 deletion(-) diff --git a/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.js b/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.js index cad0f4b805..da31ee399a 100644 --- a/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.js +++ b/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.js @@ -1,3 +1,11 @@ +/* eslint-disable + handle-callback-err, + no-dupe-keys, + no-return-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/filestore/test/unit/coffee/BucketControllerTests.js b/services/filestore/test/unit/coffee/BucketControllerTests.js index db0e6a5aa3..86db19da1b 100644 --- a/services/filestore/test/unit/coffee/BucketControllerTests.js +++ b/services/filestore/test/unit/coffee/BucketControllerTests.js @@ -1,3 +1,9 @@ +/* eslint-disable + no-return-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/filestore/test/unit/coffee/FSPersistorManagerTests.js b/services/filestore/test/unit/coffee/FSPersistorManagerTests.js index cd73f41ac0..b3c2842b46 100644 --- a/services/filestore/test/unit/coffee/FSPersistorManagerTests.js +++ b/services/filestore/test/unit/coffee/FSPersistorManagerTests.js @@ -1,3 +1,10 @@ +/* eslint-disable + handle-callback-err, + no-return-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/filestore/test/unit/coffee/FileControllerTests.js b/services/filestore/test/unit/coffee/FileControllerTests.js index ea3b0e32b8..d60ca5fbd1 100644 --- a/services/filestore/test/unit/coffee/FileControllerTests.js +++ b/services/filestore/test/unit/coffee/FileControllerTests.js @@ -1,3 +1,9 @@ +/* eslint-disable + no-return-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/filestore/test/unit/coffee/FileConverterTests.js b/services/filestore/test/unit/coffee/FileConverterTests.js index c546b61a15..96cb0ee53a 100644 --- a/services/filestore/test/unit/coffee/FileConverterTests.js +++ b/services/filestore/test/unit/coffee/FileConverterTests.js @@ -1,3 +1,10 @@ +/* eslint-disable + handle-callback-err, + no-return-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/filestore/test/unit/coffee/FileHandlerTests.js b/services/filestore/test/unit/coffee/FileHandlerTests.js index f83561166f..13c60f08da 100644 --- a/services/filestore/test/unit/coffee/FileHandlerTests.js +++ b/services/filestore/test/unit/coffee/FileHandlerTests.js @@ -1,3 +1,10 @@ +/* eslint-disable + handle-callback-err, + no-return-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/filestore/test/unit/coffee/ImageOptimiserTests.js b/services/filestore/test/unit/coffee/ImageOptimiserTests.js index 6074120a56..4e6cb858f4 100644 --- a/services/filestore/test/unit/coffee/ImageOptimiserTests.js +++ b/services/filestore/test/unit/coffee/ImageOptimiserTests.js @@ -1,3 +1,10 @@ +/* eslint-disable + handle-callback-err, + no-return-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/filestore/test/unit/coffee/KeybuilderTests.js b/services/filestore/test/unit/coffee/KeybuilderTests.js index 063a278f23..1e99899be7 100644 --- a/services/filestore/test/unit/coffee/KeybuilderTests.js +++ b/services/filestore/test/unit/coffee/KeybuilderTests.js @@ -1,3 +1,9 @@ +/* eslint-disable + no-return-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/filestore/test/unit/coffee/LocalFileWriterTests.js b/services/filestore/test/unit/coffee/LocalFileWriterTests.js index ed3eb2dfa3..1bf131a3c4 100644 --- a/services/filestore/test/unit/coffee/LocalFileWriterTests.js +++ b/services/filestore/test/unit/coffee/LocalFileWriterTests.js @@ -1,3 +1,10 @@ +/* eslint-disable + handle-callback-err, + no-return-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/filestore/test/unit/coffee/PersistorManagerTests.js b/services/filestore/test/unit/coffee/PersistorManagerTests.js index d5b859448a..620d6f0083 100644 --- a/services/filestore/test/unit/coffee/PersistorManagerTests.js +++ b/services/filestore/test/unit/coffee/PersistorManagerTests.js @@ -1,3 +1,9 @@ +/* eslint-disable + no-return-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/filestore/test/unit/coffee/S3PersistorManagerTests.js b/services/filestore/test/unit/coffee/S3PersistorManagerTests.js index 4396ce9606..a3a7d16825 100644 --- a/services/filestore/test/unit/coffee/S3PersistorManagerTests.js +++ b/services/filestore/test/unit/coffee/S3PersistorManagerTests.js @@ -1,3 +1,10 @@ +/* eslint-disable + handle-callback-err, + no-return-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/filestore/test/unit/coffee/SafeExecTests.js b/services/filestore/test/unit/coffee/SafeExecTests.js index 4c462f887d..f9a0e51ca8 100644 --- a/services/filestore/test/unit/coffee/SafeExecTests.js +++ b/services/filestore/test/unit/coffee/SafeExecTests.js @@ -1,3 +1,9 @@ +/* eslint-disable + no-return-assign, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns diff --git a/services/filestore/test/unit/coffee/SettingsTests.js b/services/filestore/test/unit/coffee/SettingsTests.js index 5c1b79d70c..98d17723a9 100644 --- a/services/filestore/test/unit/coffee/SettingsTests.js +++ b/services/filestore/test/unit/coffee/SettingsTests.js @@ -1,3 +1,9 @@ +/* eslint-disable + camelcase, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns @@ -21,7 +27,7 @@ describe("Settings", () => describe("s3", () => it("should use JSONified env var auth_secret: 'bucket1_secret' } }; - process.env['S3_BUCKET_CREDENTIALS'] = JSON.stringify(s3_settings); + process.env.S3_BUCKET_CREDENTIALS = JSON.stringify(s3_settings); const settings = require("settings-sharelatex"); expect(settings.filestore.s3BucketCreds).to.deep.equal(s3_settings); From ec60f778e62f9180ddc3c1584eaccc7372863db3 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 16 Dec 2019 11:20:27 +0000 Subject: [PATCH 25/73] Rename test/unit/coffee to test/unit/js --- .../test/unit/{coffee => js}/AWSSDKPersistorManagerTests.js | 0 .../filestore/test/unit/{coffee => js}/BucketControllerTests.js | 0 .../filestore/test/unit/{coffee => js}/FSPersistorManagerTests.js | 0 .../filestore/test/unit/{coffee => js}/FileControllerTests.js | 0 services/filestore/test/unit/{coffee => js}/FileConverterTests.js | 0 services/filestore/test/unit/{coffee => js}/FileHandlerTests.js | 0 .../filestore/test/unit/{coffee => js}/ImageOptimiserTests.js | 0 services/filestore/test/unit/{coffee => js}/KeybuilderTests.js | 0 .../filestore/test/unit/{coffee => js}/LocalFileWriterTests.js | 0 .../filestore/test/unit/{coffee => js}/PersistorManagerTests.js | 0 .../filestore/test/unit/{coffee => js}/S3PersistorManagerTests.js | 0 services/filestore/test/unit/{coffee => js}/SafeExecTests.js | 0 services/filestore/test/unit/{coffee => js}/SettingsTests.js | 0 13 files changed, 0 insertions(+), 0 deletions(-) rename services/filestore/test/unit/{coffee => js}/AWSSDKPersistorManagerTests.js (100%) rename services/filestore/test/unit/{coffee => js}/BucketControllerTests.js (100%) rename services/filestore/test/unit/{coffee => js}/FSPersistorManagerTests.js (100%) rename services/filestore/test/unit/{coffee => js}/FileControllerTests.js (100%) rename services/filestore/test/unit/{coffee => js}/FileConverterTests.js (100%) rename services/filestore/test/unit/{coffee => js}/FileHandlerTests.js (100%) rename services/filestore/test/unit/{coffee => js}/ImageOptimiserTests.js (100%) rename services/filestore/test/unit/{coffee => js}/KeybuilderTests.js (100%) rename services/filestore/test/unit/{coffee => js}/LocalFileWriterTests.js (100%) rename services/filestore/test/unit/{coffee => js}/PersistorManagerTests.js (100%) rename services/filestore/test/unit/{coffee => js}/S3PersistorManagerTests.js (100%) rename services/filestore/test/unit/{coffee => js}/SafeExecTests.js (100%) rename services/filestore/test/unit/{coffee => js}/SettingsTests.js (100%) diff --git a/services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.js b/services/filestore/test/unit/js/AWSSDKPersistorManagerTests.js similarity index 100% rename from services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.js rename to services/filestore/test/unit/js/AWSSDKPersistorManagerTests.js diff --git a/services/filestore/test/unit/coffee/BucketControllerTests.js b/services/filestore/test/unit/js/BucketControllerTests.js similarity index 100% rename from services/filestore/test/unit/coffee/BucketControllerTests.js rename to services/filestore/test/unit/js/BucketControllerTests.js diff --git a/services/filestore/test/unit/coffee/FSPersistorManagerTests.js b/services/filestore/test/unit/js/FSPersistorManagerTests.js similarity index 100% rename from services/filestore/test/unit/coffee/FSPersistorManagerTests.js rename to services/filestore/test/unit/js/FSPersistorManagerTests.js diff --git a/services/filestore/test/unit/coffee/FileControllerTests.js b/services/filestore/test/unit/js/FileControllerTests.js similarity index 100% rename from services/filestore/test/unit/coffee/FileControllerTests.js rename to services/filestore/test/unit/js/FileControllerTests.js diff --git a/services/filestore/test/unit/coffee/FileConverterTests.js b/services/filestore/test/unit/js/FileConverterTests.js similarity index 100% rename from services/filestore/test/unit/coffee/FileConverterTests.js rename to services/filestore/test/unit/js/FileConverterTests.js diff --git a/services/filestore/test/unit/coffee/FileHandlerTests.js b/services/filestore/test/unit/js/FileHandlerTests.js similarity index 100% rename from services/filestore/test/unit/coffee/FileHandlerTests.js rename to services/filestore/test/unit/js/FileHandlerTests.js diff --git a/services/filestore/test/unit/coffee/ImageOptimiserTests.js b/services/filestore/test/unit/js/ImageOptimiserTests.js similarity index 100% rename from services/filestore/test/unit/coffee/ImageOptimiserTests.js rename to services/filestore/test/unit/js/ImageOptimiserTests.js diff --git a/services/filestore/test/unit/coffee/KeybuilderTests.js b/services/filestore/test/unit/js/KeybuilderTests.js similarity index 100% rename from services/filestore/test/unit/coffee/KeybuilderTests.js rename to services/filestore/test/unit/js/KeybuilderTests.js diff --git a/services/filestore/test/unit/coffee/LocalFileWriterTests.js b/services/filestore/test/unit/js/LocalFileWriterTests.js similarity index 100% rename from services/filestore/test/unit/coffee/LocalFileWriterTests.js rename to services/filestore/test/unit/js/LocalFileWriterTests.js diff --git a/services/filestore/test/unit/coffee/PersistorManagerTests.js b/services/filestore/test/unit/js/PersistorManagerTests.js similarity index 100% rename from services/filestore/test/unit/coffee/PersistorManagerTests.js rename to services/filestore/test/unit/js/PersistorManagerTests.js diff --git a/services/filestore/test/unit/coffee/S3PersistorManagerTests.js b/services/filestore/test/unit/js/S3PersistorManagerTests.js similarity index 100% rename from services/filestore/test/unit/coffee/S3PersistorManagerTests.js rename to services/filestore/test/unit/js/S3PersistorManagerTests.js diff --git a/services/filestore/test/unit/coffee/SafeExecTests.js b/services/filestore/test/unit/js/SafeExecTests.js similarity index 100% rename from services/filestore/test/unit/coffee/SafeExecTests.js rename to services/filestore/test/unit/js/SafeExecTests.js diff --git a/services/filestore/test/unit/coffee/SettingsTests.js b/services/filestore/test/unit/js/SettingsTests.js similarity index 100% rename from services/filestore/test/unit/coffee/SettingsTests.js rename to services/filestore/test/unit/js/SettingsTests.js From eae7d284958d68ae9b0cb15cde4b5a3f561fe8aa Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 16 Dec 2019 11:20:29 +0000 Subject: [PATCH 26/73] Prettier: convert test/unit decaffeinated files to Prettier format --- .../unit/js/AWSSDKPersistorManagerTests.js | 778 ++++++++------ .../test/unit/js/BucketControllerTests.js | 174 ++-- .../test/unit/js/FSPersistorManagerTests.js | 719 +++++++------ .../test/unit/js/FileControllerTests.js | 508 ++++----- .../test/unit/js/FileConverterTests.js | 196 ++-- .../test/unit/js/FileHandlerTests.js | 578 ++++++----- .../test/unit/js/ImageOptimiserTests.js | 136 ++- .../filestore/test/unit/js/KeybuilderTests.js | 91 +- .../test/unit/js/LocalFileWriterTests.js | 208 ++-- .../test/unit/js/PersistorManagerTests.js | 242 ++--- .../test/unit/js/S3PersistorManagerTests.js | 961 ++++++++++-------- .../filestore/test/unit/js/SafeExecTests.js | 142 +-- .../filestore/test/unit/js/SettingsTests.js | 38 +- 13 files changed, 2642 insertions(+), 2129 deletions(-) diff --git a/services/filestore/test/unit/js/AWSSDKPersistorManagerTests.js b/services/filestore/test/unit/js/AWSSDKPersistorManagerTests.js index da31ee399a..ea88da71c3 100644 --- a/services/filestore/test/unit/js/AWSSDKPersistorManagerTests.js +++ b/services/filestore/test/unit/js/AWSSDKPersistorManagerTests.js @@ -11,351 +11,499 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const sinon = require('sinon'); -const chai = require('chai'); +const sinon = require('sinon') +const chai = require('chai') -const should = chai.should(); -const { - expect -} = chai; +const should = chai.should() +const { expect } = chai -const modulePath = "../../../app/js/AWSSDKPersistorManager.js"; -const SandboxedModule = require('sandboxed-module'); +const modulePath = '../../../app/js/AWSSDKPersistorManager.js' +const SandboxedModule = require('sandboxed-module') -describe("AWSSDKPersistorManager", function() { - beforeEach(function() { - this.settings = { - filestore: { - backend: "aws-sdk" - } - }; - this.s3 = { - upload: sinon.stub(), - getObject: sinon.stub(), - copyObject: sinon.stub(), - deleteObject: sinon.stub(), - listObjects: sinon.stub(), - deleteObjects: sinon.stub(), - headObject: sinon.stub() - }; - this.awssdk = - {S3: sinon.stub().returns(this.s3)}; +describe('AWSSDKPersistorManager', function() { + beforeEach(function() { + this.settings = { + filestore: { + backend: 'aws-sdk' + } + } + this.s3 = { + upload: sinon.stub(), + getObject: sinon.stub(), + copyObject: sinon.stub(), + deleteObject: sinon.stub(), + listObjects: sinon.stub(), + deleteObjects: sinon.stub(), + headObject: sinon.stub() + } + this.awssdk = { S3: sinon.stub().returns(this.s3) } - this.requires = { - "aws-sdk": this.awssdk, - "settings-sharelatex": this.settings, - "logger-sharelatex": { - log() {}, - err() {} - }, - "fs": (this.fs = - {createReadStream: sinon.stub()}), - "./Errors": (this.Errors = - {NotFoundError: sinon.stub()}) - }; - this.key = "my/key"; - this.bucketName = "my-bucket"; - this.error = "my error"; - return this.AWSSDKPersistorManager = SandboxedModule.require(modulePath, {requires: this.requires}); - }); + this.requires = { + 'aws-sdk': this.awssdk, + 'settings-sharelatex': this.settings, + 'logger-sharelatex': { + log() {}, + err() {} + }, + fs: (this.fs = { createReadStream: sinon.stub() }), + './Errors': (this.Errors = { NotFoundError: sinon.stub() }) + } + this.key = 'my/key' + this.bucketName = 'my-bucket' + this.error = 'my error' + return (this.AWSSDKPersistorManager = SandboxedModule.require(modulePath, { + requires: this.requires + })) + }) - describe("sendFile", function() { - beforeEach(function() { - this.stream = {}; - this.fsPath = "/usr/local/some/file"; - return this.fs.createReadStream.returns(this.stream); - }); + describe('sendFile', function() { + beforeEach(function() { + this.stream = {} + this.fsPath = '/usr/local/some/file' + return this.fs.createReadStream.returns(this.stream) + }) - it("should put the file with s3.upload", function(done) { - this.s3.upload.callsArgWith(1); - return this.AWSSDKPersistorManager.sendFile(this.bucketName, this.key, this.fsPath, err => { - expect(err).to.not.be.ok; - expect(this.s3.upload.calledOnce, "called only once").to.be.true; - expect((this.s3.upload.calledWith({Bucket: this.bucketName, Key: this.key, Body: this.stream})) - , "called with correct arguments").to.be.true; - return done(); - }); - }); + it('should put the file with s3.upload', function(done) { + this.s3.upload.callsArgWith(1) + return this.AWSSDKPersistorManager.sendFile( + this.bucketName, + this.key, + this.fsPath, + err => { + expect(err).to.not.be.ok + expect(this.s3.upload.calledOnce, 'called only once').to.be.true + expect( + this.s3.upload.calledWith({ + Bucket: this.bucketName, + Key: this.key, + Body: this.stream + }), + 'called with correct arguments' + ).to.be.true + return done() + } + ) + }) - return it("should dispatch the error from s3.upload", function(done) { - this.s3.upload.callsArgWith(1, this.error); - return this.AWSSDKPersistorManager.sendFile(this.bucketName, this.key, this.fsPath, err => { - expect(err).to.equal(this.error); - return done(); - }); - }); - }); + return it('should dispatch the error from s3.upload', function(done) { + this.s3.upload.callsArgWith(1, this.error) + return this.AWSSDKPersistorManager.sendFile( + this.bucketName, + this.key, + this.fsPath, + err => { + expect(err).to.equal(this.error) + return done() + } + ) + }) + }) + describe('sendStream', function() { + beforeEach(function() { + return (this.stream = {}) + }) - describe("sendStream", function() { - beforeEach(function() { - return this.stream = {};}); + it('should put the file with s3.upload', function(done) { + this.s3.upload.callsArgWith(1) + return this.AWSSDKPersistorManager.sendStream( + this.bucketName, + this.key, + this.stream, + err => { + expect(err).to.not.be.ok + expect(this.s3.upload.calledOnce, 'called only once').to.be.true + expect( + this.s3.upload.calledWith({ + Bucket: this.bucketName, + Key: this.key, + Body: this.stream + }), + 'called with correct arguments' + ).to.be.true + return done() + } + ) + }) - it("should put the file with s3.upload", function(done) { - this.s3.upload.callsArgWith(1); - return this.AWSSDKPersistorManager.sendStream(this.bucketName, this.key, this.stream, err => { - expect(err).to.not.be.ok; - expect(this.s3.upload.calledOnce, "called only once").to.be.true; - expect((this.s3.upload.calledWith({Bucket: this.bucketName, Key: this.key, Body: this.stream})), - "called with correct arguments").to.be.true; - return done(); - }); - }); + return it('should dispatch the error from s3.upload', function(done) { + this.s3.upload.callsArgWith(1, this.error) + return this.AWSSDKPersistorManager.sendStream( + this.bucketName, + this.key, + this.stream, + err => { + expect(err).to.equal(this.error) + return done() + } + ) + }) + }) - return it("should dispatch the error from s3.upload", function(done) { - this.s3.upload.callsArgWith(1, this.error); - return this.AWSSDKPersistorManager.sendStream(this.bucketName, this.key, this.stream, err => { - expect(err).to.equal(this.error); - return done(); - }); - }); - }); + describe('getFileStream', function() { + beforeEach(function() { + this.opts = {} + this.stream = {} + this.read_stream = { on: (this.read_stream_on = sinon.stub()) } + this.object = { createReadStream: sinon.stub().returns(this.read_stream) } + return this.s3.getObject.returns(this.object) + }) - describe("getFileStream", function() { - beforeEach(function() { - this.opts = {}; - this.stream = {}; - this.read_stream = - {on: (this.read_stream_on = sinon.stub())}; - this.object = - {createReadStream: sinon.stub().returns(this.read_stream)}; - return this.s3.getObject.returns(this.object); - }); + it('should return a stream from s3.getObject', function(done) { + this.read_stream_on.withArgs('readable').callsArgWith(1) - it("should return a stream from s3.getObject", function(done) { - this.read_stream_on.withArgs('readable').callsArgWith(1); + return this.AWSSDKPersistorManager.getFileStream( + this.bucketName, + this.key, + this.opts, + (err, stream) => { + expect(this.read_stream_on.calledTwice) + expect(err).to.not.be.ok + expect(stream, 'returned the stream').to.equal(this.read_stream) + expect( + this.s3.getObject.calledWith({ + Bucket: this.bucketName, + Key: this.key + }), + 'called with correct arguments' + ).to.be.true + return done() + } + ) + }) - return this.AWSSDKPersistorManager.getFileStream(this.bucketName, this.key, this.opts, (err, stream) => { - expect(this.read_stream_on.calledTwice); - expect(err).to.not.be.ok; - expect(stream, "returned the stream").to.equal(this.read_stream); - expect((this.s3.getObject.calledWith({Bucket: this.bucketName, Key: this.key})), - "called with correct arguments").to.be.true; - return done(); - }); - }); + describe('with start and end options', function() { + beforeEach(function() { + return (this.opts = { + start: 0, + end: 8 + }) + }) + return it('should pass headers to the s3.GetObject', function(done) { + this.read_stream_on.withArgs('readable').callsArgWith(1) + this.AWSSDKPersistorManager.getFileStream( + this.bucketName, + this.key, + this.opts, + (err, stream) => { + return expect( + this.s3.getObject.calledWith({ + Bucket: this.bucketName, + Key: this.key, + Range: 'bytes=0-8' + }), + 'called with correct arguments' + ).to.be.true + } + ) + return done() + }) + }) - describe("with start and end options", function() { - beforeEach(function() { - return this.opts = { - start: 0, - end: 8 - }; - }); - return it("should pass headers to the s3.GetObject", function(done) { - this.read_stream_on.withArgs('readable').callsArgWith(1); - this.AWSSDKPersistorManager.getFileStream(this.bucketName, this.key, this.opts, (err, stream) => { - return expect((this.s3.getObject.calledWith({Bucket: this.bucketName, Key: this.key, Range: 'bytes=0-8'})), - "called with correct arguments").to.be.true; - }); - return done(); - }); - }); + return describe('error conditions', function() { + describe("when the file doesn't exist", function() { + beforeEach(function() { + this.error = new Error() + return (this.error.code = 'NoSuchKey') + }) + return it('should produce a NotFoundError', function(done) { + this.read_stream_on.withArgs('error').callsArgWith(1, this.error) + return this.AWSSDKPersistorManager.getFileStream( + this.bucketName, + this.key, + this.opts, + (err, stream) => { + expect(stream).to.not.be.ok + expect(err).to.be.ok + expect( + err instanceof this.Errors.NotFoundError, + 'error is a correct instance' + ).to.equal(true) + return done() + } + ) + }) + }) - return describe("error conditions", function() { - describe("when the file doesn't exist", function() { - beforeEach(function() { - this.error = new Error(); - return this.error.code = 'NoSuchKey'; - }); - return it("should produce a NotFoundError", function(done) { - this.read_stream_on.withArgs('error').callsArgWith(1, this.error); - return this.AWSSDKPersistorManager.getFileStream(this.bucketName, this.key, this.opts, (err, stream) => { - expect(stream).to.not.be.ok; - expect(err).to.be.ok; - expect(err instanceof this.Errors.NotFoundError, "error is a correct instance").to.equal(true); - return done(); - }); - }); - }); + return describe('when there is some other error', function() { + beforeEach(function() { + return (this.error = new Error()) + }) + return it('should dispatch the error from s3 object stream', function(done) { + this.read_stream_on.withArgs('error').callsArgWith(1, this.error) + return this.AWSSDKPersistorManager.getFileStream( + this.bucketName, + this.key, + this.opts, + (err, stream) => { + expect(stream).to.not.be.ok + expect(err).to.be.ok + expect(err).to.equal(this.error) + return done() + } + ) + }) + }) + }) + }) - return describe("when there is some other error", function() { - beforeEach(function() { - return this.error = new Error(); - }); - return it("should dispatch the error from s3 object stream", function(done) { - this.read_stream_on.withArgs('error').callsArgWith(1, this.error); - return this.AWSSDKPersistorManager.getFileStream(this.bucketName, this.key, this.opts, (err, stream) => { - expect(stream).to.not.be.ok; - expect(err).to.be.ok; - expect(err).to.equal(this.error); - return done(); - }); - }); - }); - }); - }); + describe('copyFile', function() { + beforeEach(function() { + this.destKey = 'some/key' + return (this.stream = {}) + }) - describe("copyFile", function() { - beforeEach(function() { - this.destKey = "some/key"; - return this.stream = {};}); + it('should copy the file with s3.copyObject', function(done) { + this.s3.copyObject.callsArgWith(1) + return this.AWSSDKPersistorManager.copyFile( + this.bucketName, + this.key, + this.destKey, + err => { + expect(err).to.not.be.ok + expect(this.s3.copyObject.calledOnce, 'called only once').to.be.true + expect( + this.s3.copyObject.calledWith({ + Bucket: this.bucketName, + Key: this.destKey, + CopySource: this.bucketName + '/' + this.key + }), + 'called with correct arguments' + ).to.be.true + return done() + } + ) + }) - it("should copy the file with s3.copyObject", function(done) { - this.s3.copyObject.callsArgWith(1); - return this.AWSSDKPersistorManager.copyFile(this.bucketName, this.key, this.destKey, err => { - expect(err).to.not.be.ok; - expect(this.s3.copyObject.calledOnce, "called only once").to.be.true; - expect((this.s3.copyObject.calledWith({Bucket: this.bucketName, Key: this.destKey, CopySource: this.bucketName + '/' + this.key})), - "called with correct arguments").to.be.true; - return done(); - }); - }); + return it('should dispatch the error from s3.copyObject', function(done) { + this.s3.copyObject.callsArgWith(1, this.error) + return this.AWSSDKPersistorManager.copyFile( + this.bucketName, + this.key, + this.destKey, + err => { + expect(err).to.equal(this.error) + return done() + } + ) + }) + }) - return it("should dispatch the error from s3.copyObject", function(done) { - this.s3.copyObject.callsArgWith(1, this.error); - return this.AWSSDKPersistorManager.copyFile(this.bucketName, this.key, this.destKey, err => { - expect(err).to.equal(this.error); - return done(); - }); - }); - }); + describe('deleteFile', function() { + it('should delete the file with s3.deleteObject', function(done) { + this.s3.deleteObject.callsArgWith(1) + return this.AWSSDKPersistorManager.deleteFile( + this.bucketName, + this.key, + err => { + expect(err).to.not.be.ok + expect(this.s3.deleteObject.calledOnce, 'called only once').to.be.true + expect( + this.s3.deleteObject.calledWith({ + Bucket: this.bucketName, + Key: this.key + }), + 'called with correct arguments' + ).to.be.true + return done() + } + ) + }) - describe("deleteFile", function() { - it("should delete the file with s3.deleteObject", function(done) { - this.s3.deleteObject.callsArgWith(1); - return this.AWSSDKPersistorManager.deleteFile(this.bucketName, this.key, err => { - expect(err).to.not.be.ok; - expect(this.s3.deleteObject.calledOnce, "called only once").to.be.true; - expect((this.s3.deleteObject.calledWith({Bucket: this.bucketName, Key: this.key})), - "called with correct arguments").to.be.true; - return done(); - }); - }); + return it('should dispatch the error from s3.deleteObject', function(done) { + this.s3.deleteObject.callsArgWith(1, this.error) + return this.AWSSDKPersistorManager.deleteFile( + this.bucketName, + this.key, + err => { + expect(err).to.equal(this.error) + return done() + } + ) + }) + }) - return it("should dispatch the error from s3.deleteObject", function(done) { - this.s3.deleteObject.callsArgWith(1, this.error); - return this.AWSSDKPersistorManager.deleteFile(this.bucketName, this.key, err => { - expect(err).to.equal(this.error); - return done(); - }); - }); - }); + describe('deleteDirectory', function() { + it('should list the directory content using s3.listObjects', function(done) { + this.s3.listObjects.callsArgWith(1, null, { Contents: [] }) + return this.AWSSDKPersistorManager.deleteDirectory( + this.bucketName, + this.key, + err => { + expect(err).to.not.be.ok + expect(this.s3.listObjects.calledOnce, 'called only once').to.be.true + expect( + this.s3.listObjects.calledWith({ + Bucket: this.bucketName, + Prefix: this.key + }), + 'called with correct arguments' + ).to.be.true + return done() + } + ) + }) - describe("deleteDirectory", function() { + it('should dispatch the error from s3.listObjects', function(done) { + this.s3.listObjects.callsArgWith(1, this.error) + return this.AWSSDKPersistorManager.deleteDirectory( + this.bucketName, + this.key, + err => { + expect(err).to.equal(this.error) + return done() + } + ) + }) - it("should list the directory content using s3.listObjects", function(done) { - this.s3.listObjects.callsArgWith(1, null, {Contents: []}); - return this.AWSSDKPersistorManager.deleteDirectory(this.bucketName, this.key, err => { - expect(err).to.not.be.ok; - expect(this.s3.listObjects.calledOnce, "called only once").to.be.true; - expect((this.s3.listObjects.calledWith({Bucket: this.bucketName, Prefix: this.key})), - "called with correct arguments").to.be.true; - return done(); - }); - }); + return describe('with directory content', function() { + beforeEach(function() { + return (this.fileList = [{ Key: 'foo' }, { Key: 'bar', Key: 'baz' }]) + }) - it("should dispatch the error from s3.listObjects", function(done) { - this.s3.listObjects.callsArgWith(1, this.error); - return this.AWSSDKPersistorManager.deleteDirectory(this.bucketName, this.key, err => { - expect(err).to.equal(this.error); - return done(); - }); - }); + it('should forward the file keys to s3.deleteObjects', function(done) { + this.s3.listObjects.callsArgWith(1, null, { Contents: this.fileList }) + this.s3.deleteObjects.callsArgWith(1) + return this.AWSSDKPersistorManager.deleteDirectory( + this.bucketName, + this.key, + err => { + expect(err).to.not.be.ok + expect(this.s3.deleteObjects.calledOnce, 'called only once').to.be + .true + expect( + this.s3.deleteObjects.calledWith({ + Bucket: this.bucketName, + Delete: { + Quiet: true, + Objects: this.fileList + } + }), + 'called with correct arguments' + ).to.be.true + return done() + } + ) + }) - return describe("with directory content", function() { - beforeEach(function() { - return this.fileList = [ - {Key: 'foo'} - , { Key: 'bar' - , Key: 'baz' - } - ];}); + return it('should dispatch the error from s3.deleteObjects', function(done) { + this.s3.listObjects.callsArgWith(1, null, { Contents: this.fileList }) + this.s3.deleteObjects.callsArgWith(1, this.error) + return this.AWSSDKPersistorManager.deleteDirectory( + this.bucketName, + this.key, + err => { + expect(err).to.equal(this.error) + return done() + } + ) + }) + }) + }) - it("should forward the file keys to s3.deleteObjects", function(done) { - this.s3.listObjects.callsArgWith(1, null, {Contents: this.fileList}); - this.s3.deleteObjects.callsArgWith(1); - return this.AWSSDKPersistorManager.deleteDirectory(this.bucketName, this.key, err => { - expect(err).to.not.be.ok; - expect(this.s3.deleteObjects.calledOnce, "called only once").to.be.true; - expect((this.s3.deleteObjects.calledWith({ - Bucket: this.bucketName, - Delete: { - Quiet: true, - Objects: this.fileList - }})), - "called with correct arguments").to.be.true; - return done(); - }); - }); + describe('checkIfFileExists', function() { + it('should check for the file with s3.headObject', function(done) { + this.s3.headObject.callsArgWith(1, null, {}) + return this.AWSSDKPersistorManager.checkIfFileExists( + this.bucketName, + this.key, + (err, exists) => { + expect(err).to.not.be.ok + expect(this.s3.headObject.calledOnce, 'called only once').to.be.true + expect( + this.s3.headObject.calledWith({ + Bucket: this.bucketName, + Key: this.key + }), + 'called with correct arguments' + ).to.be.true + return done() + } + ) + }) - return it("should dispatch the error from s3.deleteObjects", function(done) { - this.s3.listObjects.callsArgWith(1, null, {Contents: this.fileList}); - this.s3.deleteObjects.callsArgWith(1, this.error); - return this.AWSSDKPersistorManager.deleteDirectory(this.bucketName, this.key, err => { - expect(err).to.equal(this.error); - return done(); - }); - }); - }); - }); + it('should return false on an inexistant file', function(done) { + this.s3.headObject.callsArgWith(1, null, {}) + return this.AWSSDKPersistorManager.checkIfFileExists( + this.bucketName, + this.key, + (err, exists) => { + expect(exists).to.be.false + return done() + } + ) + }) + it('should return true on an existing file', function(done) { + this.s3.headObject.callsArgWith(1, null, { ETag: 'etag' }) + return this.AWSSDKPersistorManager.checkIfFileExists( + this.bucketName, + this.key, + (err, exists) => { + expect(exists).to.be.true + return done() + } + ) + }) - describe("checkIfFileExists", function() { + return it('should dispatch the error from s3.headObject', function(done) { + this.s3.headObject.callsArgWith(1, this.error) + return this.AWSSDKPersistorManager.checkIfFileExists( + this.bucketName, + this.key, + (err, exists) => { + expect(err).to.equal(this.error) + return done() + } + ) + }) + }) - it("should check for the file with s3.headObject", function(done) { - this.s3.headObject.callsArgWith(1, null, {}); - return this.AWSSDKPersistorManager.checkIfFileExists(this.bucketName, this.key, (err, exists) => { - expect(err).to.not.be.ok; - expect(this.s3.headObject.calledOnce, "called only once").to.be.true; - expect((this.s3.headObject.calledWith({Bucket: this.bucketName, Key: this.key})), - "called with correct arguments").to.be.true; - return done(); - }); - }); + return describe('directorySize', function() { + it('should list the directory content using s3.listObjects', function(done) { + this.s3.listObjects.callsArgWith(1, null, { Contents: [] }) + return this.AWSSDKPersistorManager.directorySize( + this.bucketName, + this.key, + err => { + expect(err).to.not.be.ok + expect(this.s3.listObjects.calledOnce, 'called only once').to.be.true + expect( + this.s3.listObjects.calledWith({ + Bucket: this.bucketName, + Prefix: this.key + }), + 'called with correct arguments' + ).to.be.true + return done() + } + ) + }) - it("should return false on an inexistant file", function(done) { - this.s3.headObject.callsArgWith(1, null, {}); - return this.AWSSDKPersistorManager.checkIfFileExists(this.bucketName, this.key, (err, exists) => { - expect(exists).to.be.false; - return done(); - }); - }); + it('should dispatch the error from s3.listObjects', function(done) { + this.s3.listObjects.callsArgWith(1, this.error) + return this.AWSSDKPersistorManager.directorySize( + this.bucketName, + this.key, + err => { + expect(err).to.equal(this.error) + return done() + } + ) + }) - it("should return true on an existing file", function(done) { - this.s3.headObject.callsArgWith(1, null, {ETag: "etag"}); - return this.AWSSDKPersistorManager.checkIfFileExists(this.bucketName, this.key, (err, exists) => { - expect(exists).to.be.true; - return done(); - }); - }); - - return it("should dispatch the error from s3.headObject", function(done) { - this.s3.headObject.callsArgWith(1, this.error); - return this.AWSSDKPersistorManager.checkIfFileExists(this.bucketName, this.key, (err, exists) => { - expect(err).to.equal(this.error); - return done(); - }); - }); - }); - - return describe("directorySize", function() { - - it("should list the directory content using s3.listObjects", function(done) { - this.s3.listObjects.callsArgWith(1, null, {Contents: []}); - return this.AWSSDKPersistorManager.directorySize(this.bucketName, this.key, err => { - expect(err).to.not.be.ok; - expect(this.s3.listObjects.calledOnce, "called only once").to.be.true; - expect((this.s3.listObjects.calledWith({Bucket: this.bucketName, Prefix: this.key})), - "called with correct arguments").to.be.true; - return done(); - }); - }); - - it("should dispatch the error from s3.listObjects", function(done) { - this.s3.listObjects.callsArgWith(1, this.error); - return this.AWSSDKPersistorManager.directorySize(this.bucketName, this.key, err => { - expect(err).to.equal(this.error); - return done(); - }); - }); - - return it("should sum directory files sizes", function(done) { - this.s3.listObjects.callsArgWith(1, null, {Contents: [ { Size: 1024 }, { Size: 2048 }]}); - return this.AWSSDKPersistorManager.directorySize(this.bucketName, this.key, (err, size) => { - expect(size).to.equal(3072); - return done(); - }); - }); - }); -}); + return it('should sum directory files sizes', function(done) { + this.s3.listObjects.callsArgWith(1, null, { + Contents: [{ Size: 1024 }, { Size: 2048 }] + }) + return this.AWSSDKPersistorManager.directorySize( + this.bucketName, + this.key, + (err, size) => { + expect(size).to.equal(3072) + return done() + } + ) + }) + }) +}) diff --git a/services/filestore/test/unit/js/BucketControllerTests.js b/services/filestore/test/unit/js/BucketControllerTests.js index 86db19da1b..ef74b3f6c0 100644 --- a/services/filestore/test/unit/js/BucketControllerTests.js +++ b/services/filestore/test/unit/js/BucketControllerTests.js @@ -9,98 +9,92 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const { - assert -} = require("chai"); -const sinon = require('sinon'); -const chai = require('chai'); -const should = chai.should(); -const { - expect -} = chai; -const modulePath = "../../../app/js/BucketController.js"; -const SandboxedModule = require('sandboxed-module'); +const { assert } = require('chai') +const sinon = require('sinon') +const chai = require('chai') +const should = chai.should() +const { expect } = chai +const modulePath = '../../../app/js/BucketController.js' +const SandboxedModule = require('sandboxed-module') -describe("BucketController", function() { +describe('BucketController', function() { + beforeEach(function() { + this.PersistorManager = { + sendStream: sinon.stub(), + copyFile: sinon.stub(), + deleteFile: sinon.stub() + } - beforeEach(function() { - this.PersistorManager = { - sendStream: sinon.stub(), - copyFile: sinon.stub(), - deleteFile:sinon.stub() - }; + this.settings = { + s3: { + buckets: { + user_files: 'user_files' + } + }, + filestore: { + backend: 's3', + s3: { + secret: 'secret', + key: 'this_key' + } + } + } - this.settings = { - s3: { - buckets: { - user_files:"user_files" - } - }, - filestore: { - backend: "s3", - s3: { - secret: "secret", - key: "this_key" - } - } - }; + this.FileHandler = { + getFile: sinon.stub(), + deleteFile: sinon.stub(), + insertFile: sinon.stub(), + getDirectorySize: sinon.stub() + } + this.LocalFileWriter = {} + this.controller = SandboxedModule.require(modulePath, { + requires: { + './LocalFileWriter': this.LocalFileWriter, + './FileHandler': this.FileHandler, + './PersistorManager': this.PersistorManager, + 'settings-sharelatex': this.settings, + 'metrics-sharelatex': { + inc() {} + }, + 'logger-sharelatex': { + log() {}, + err() {} + } + } + }) + this.project_id = 'project_id' + this.file_id = 'file_id' + this.bucket = 'user_files' + this.key = `${this.project_id}/${this.file_id}` + this.req = { + query: {}, + params: { + bucket: this.bucket, + 0: this.key + }, + headers: {} + } + this.res = { setHeader() {} } + return (this.fileStream = {}) + }) - this.FileHandler = { - getFile: sinon.stub(), - deleteFile: sinon.stub(), - insertFile: sinon.stub(), - getDirectorySize: sinon.stub() - }; - this.LocalFileWriter = {}; - this.controller = SandboxedModule.require(modulePath, { requires: { - "./LocalFileWriter":this.LocalFileWriter, - "./FileHandler": this.FileHandler, - "./PersistorManager":this.PersistorManager, - "settings-sharelatex": this.settings, - "metrics-sharelatex": { - inc() {} - }, - "logger-sharelatex": { - log() {}, - err() {} - } - } - } - ); - this.project_id = "project_id"; - this.file_id = "file_id"; - this.bucket = "user_files"; - this.key = `${this.project_id}/${this.file_id}`; - this.req = { - query:{}, - params: { - bucket: this.bucket, - 0: this.key - }, - headers: {} - }; - this.res = - {setHeader() {}}; - return this.fileStream = {};}); + return describe('getFile', function() { + it('should pipe the stream', function(done) { + this.FileHandler.getFile.callsArgWith(3, null, this.fileStream) + this.fileStream.pipe = res => { + res.should.equal(this.res) + return done() + } + return this.controller.getFile(this.req, this.res) + }) - return describe("getFile", function() { - - it("should pipe the stream", function(done){ - this.FileHandler.getFile.callsArgWith(3, null, this.fileStream); - this.fileStream.pipe = res=> { - res.should.equal(this.res); - return done(); - }; - return this.controller.getFile(this.req, this.res); - }); - - return it("should send a 500 if there is a problem", function(done){ - this.FileHandler.getFile.callsArgWith(3, "error"); - this.res.send = code=> { - code.should.equal(500); - return done(); - }; - return this.controller.getFile(this.req, this.res); - }); - }); -}); + return it('should send a 500 if there is a problem', function(done) { + this.FileHandler.getFile.callsArgWith(3, 'error') + this.res.send = code => { + code.should.equal(500) + return done() + } + return this.controller.getFile(this.req, this.res) + }) + }) +}) diff --git a/services/filestore/test/unit/js/FSPersistorManagerTests.js b/services/filestore/test/unit/js/FSPersistorManagerTests.js index b3c2842b46..9e9018c17b 100644 --- a/services/filestore/test/unit/js/FSPersistorManagerTests.js +++ b/services/filestore/test/unit/js/FSPersistorManagerTests.js @@ -11,369 +11,492 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const { - assert -} = require("chai"); -const sinon = require('sinon'); -const chai = require('chai'); -const { - should -} = chai; -const { - expect -} = chai; -const modulePath = "../../../app/js/FSPersistorManager.js"; -const SandboxedModule = require('sandboxed-module'); -const fs = require("fs"); -const response = require("response"); - -describe("FSPersistorManagerTests", function() { +const { assert } = require('chai') +const sinon = require('sinon') +const chai = require('chai') +const { should } = chai +const { expect } = chai +const modulePath = '../../../app/js/FSPersistorManager.js' +const SandboxedModule = require('sandboxed-module') +const fs = require('fs') +const response = require('response') +describe('FSPersistorManagerTests', function() { beforeEach(function() { this.Fs = { - rename:sinon.stub(), - createReadStream:sinon.stub(), - createWriteStream:sinon.stub(), - unlink:sinon.stub(), - rmdir:sinon.stub(), - exists:sinon.stub(), - readdir:sinon.stub(), - open:sinon.stub(), - openSync:sinon.stub(), - fstatSync:sinon.stub(), - closeSync:sinon.stub(), - stat:sinon.stub() - }; - this.Rimraf = sinon.stub(); + rename: sinon.stub(), + createReadStream: sinon.stub(), + createWriteStream: sinon.stub(), + unlink: sinon.stub(), + rmdir: sinon.stub(), + exists: sinon.stub(), + readdir: sinon.stub(), + open: sinon.stub(), + openSync: sinon.stub(), + fstatSync: sinon.stub(), + closeSync: sinon.stub(), + stat: sinon.stub() + } + this.Rimraf = sinon.stub() this.LocalFileWriter = { writeStream: sinon.stub(), deleteFile: sinon.stub() - }; + } this.requires = { - "./LocalFileWriter":this.LocalFileWriter, - "fs":this.Fs, - "logger-sharelatex": { + './LocalFileWriter': this.LocalFileWriter, + fs: this.Fs, + 'logger-sharelatex': { log() {}, err() {} }, - "response":response, - "rimraf":this.Rimraf, - "./Errors": (this.Errors = - {NotFoundError: sinon.stub()}) - }; - this.location = "/tmp"; - this.name1 = "530f2407e7ef165704000007/530f838b46d9a9e859000008"; - this.name1Filtered ="530f2407e7ef165704000007_530f838b46d9a9e859000008"; - this.name2 = "second_file"; - this.error = "error_message"; - return this.FSPersistorManager = SandboxedModule.require(modulePath, {requires: this.requires}); - }); + response: response, + rimraf: this.Rimraf, + './Errors': (this.Errors = { NotFoundError: sinon.stub() }) + } + this.location = '/tmp' + this.name1 = '530f2407e7ef165704000007/530f838b46d9a9e859000008' + this.name1Filtered = '530f2407e7ef165704000007_530f838b46d9a9e859000008' + this.name2 = 'second_file' + this.error = 'error_message' + return (this.FSPersistorManager = SandboxedModule.require(modulePath, { + requires: this.requires + })) + }) - describe("sendFile", function() { + describe('sendFile', function() { beforeEach(function() { - return this.Fs.createReadStream = sinon.stub().returns({ + return (this.Fs.createReadStream = sinon.stub().returns({ on() {}, pipe() {} - }); - }); + })) + }) - it("should copy the file", function(done) { - this.Fs.createWriteStream =sinon.stub().returns({ + it('should copy the file', function(done) { + this.Fs.createWriteStream = sinon.stub().returns({ on(event, handler) { - if (event === 'finish') { return process.nextTick(handler); } + if (event === 'finish') { + return process.nextTick(handler) + } } - }); - return this.FSPersistorManager.sendFile(this.location, this.name1, this.name2, err=> { - this.Fs.createReadStream.calledWith(this.name2).should.equal(true); - this.Fs.createWriteStream.calledWith(`${this.location}/${this.name1Filtered}` ).should.equal(true); - return done(); - }); - }); + }) + return this.FSPersistorManager.sendFile( + this.location, + this.name1, + this.name2, + err => { + this.Fs.createReadStream.calledWith(this.name2).should.equal(true) + this.Fs.createWriteStream + .calledWith(`${this.location}/${this.name1Filtered}`) + .should.equal(true) + return done() + } + ) + }) - return it("should return an error if the file cannot be stored", function(done) { - this.Fs.createWriteStream =sinon.stub().returns({ + return it('should return an error if the file cannot be stored', function(done) { + this.Fs.createWriteStream = sinon.stub().returns({ on: (event, handler) => { - if (event === 'error') { - return process.nextTick(() => { - return handler(this.error); - }); + if (event === 'error') { + return process.nextTick(() => { + return handler(this.error) + }) + } } - } - }); - return this.FSPersistorManager.sendFile(this.location, this.name1, this.name2, err=> { - this.Fs.createReadStream.calledWith(this.name2).should.equal(true); - this.Fs.createWriteStream.calledWith(`${this.location}/${this.name1Filtered}` ).should.equal(true); - err.should.equal(this.error); - return done(); - }); - }); - }); + }) + return this.FSPersistorManager.sendFile( + this.location, + this.name1, + this.name2, + err => { + this.Fs.createReadStream.calledWith(this.name2).should.equal(true) + this.Fs.createWriteStream + .calledWith(`${this.location}/${this.name1Filtered}`) + .should.equal(true) + err.should.equal(this.error) + return done() + } + ) + }) + }) - describe("sendStream", function() { + describe('sendStream', function() { beforeEach(function() { - this.FSPersistorManager.sendFile = sinon.stub().callsArgWith(3); - this.LocalFileWriter.writeStream.callsArgWith(2, null, this.name1); - this.LocalFileWriter.deleteFile.callsArg(1); - return this.SourceStream = - {on() {}}; - }); + this.FSPersistorManager.sendFile = sinon.stub().callsArgWith(3) + this.LocalFileWriter.writeStream.callsArgWith(2, null, this.name1) + this.LocalFileWriter.deleteFile.callsArg(1) + return (this.SourceStream = { on() {} }) + }) - it("should sent stream to LocalFileWriter", function(done){ - return this.FSPersistorManager.sendStream(this.location, this.name1, this.SourceStream, () => { - this.LocalFileWriter.writeStream.calledWith(this.SourceStream).should.equal(true); - return done(); - }); - }); + it('should sent stream to LocalFileWriter', function(done) { + return this.FSPersistorManager.sendStream( + this.location, + this.name1, + this.SourceStream, + () => { + this.LocalFileWriter.writeStream + .calledWith(this.SourceStream) + .should.equal(true) + return done() + } + ) + }) - it("should return the error from LocalFileWriter", function(done){ - this.LocalFileWriter.writeStream.callsArgWith(2, this.error); - return this.FSPersistorManager.sendStream(this.location, this.name1, this.SourceStream, err=> { - err.should.equal(this.error); - return done(); - }); - }); + it('should return the error from LocalFileWriter', function(done) { + this.LocalFileWriter.writeStream.callsArgWith(2, this.error) + return this.FSPersistorManager.sendStream( + this.location, + this.name1, + this.SourceStream, + err => { + err.should.equal(this.error) + return done() + } + ) + }) - return it("should send the file to the filestore", function(done){ - this.LocalFileWriter.writeStream.callsArgWith(2); - return this.FSPersistorManager.sendStream(this.location, this.name1, this.SourceStream, err=> { - this.FSPersistorManager.sendFile.called.should.equal(true); - return done(); - }); - }); - }); + return it('should send the file to the filestore', function(done) { + this.LocalFileWriter.writeStream.callsArgWith(2) + return this.FSPersistorManager.sendStream( + this.location, + this.name1, + this.SourceStream, + err => { + this.FSPersistorManager.sendFile.called.should.equal(true) + return done() + } + ) + }) + }) - describe("getFileStream", function() { + describe('getFileStream', function() { beforeEach(function() { - return this.opts = {};}); + return (this.opts = {}) + }) - it("should use correct file location", function(done) { - this.FSPersistorManager.getFileStream(this.location, this.name1, this.opts, (err,res) => {}); - this.Fs.open.calledWith(`${this.location}/${this.name1Filtered}`).should.equal(true); - return done(); - }); - - describe("with start and end options", function() { + it('should use correct file location', function(done) { + this.FSPersistorManager.getFileStream( + this.location, + this.name1, + this.opts, + (err, res) => {} + ) + this.Fs.open + .calledWith(`${this.location}/${this.name1Filtered}`) + .should.equal(true) + return done() + }) + describe('with start and end options', function() { beforeEach(function() { - this.fd = 2019; - this.opts_in = {start: 0, end: 8}; - this.opts = {start: 0, end: 8, fd: this.fd}; - return this.Fs.open.callsArgWith(2, null, this.fd); - }); + this.fd = 2019 + this.opts_in = { start: 0, end: 8 } + this.opts = { start: 0, end: 8, fd: this.fd } + return this.Fs.open.callsArgWith(2, null, this.fd) + }) return it('should pass the options to createReadStream', function(done) { - this.FSPersistorManager.getFileStream(this.location, this.name1, this.opts_in, (err,res)=> {}); - this.Fs.createReadStream.calledWith(null, this.opts).should.equal(true); - return done(); - }); - }); - - return describe("error conditions", function() { - - describe("when the file does not exist", function() { + this.FSPersistorManager.getFileStream( + this.location, + this.name1, + this.opts_in, + (err, res) => {} + ) + this.Fs.createReadStream.calledWith(null, this.opts).should.equal(true) + return done() + }) + }) + return describe('error conditions', function() { + describe('when the file does not exist', function() { beforeEach(function() { - this.fakeCode = 'ENOENT'; - const err = new Error(); - err.code = this.fakeCode; - return this.Fs.open.callsArgWith(2, err, null); - }); + this.fakeCode = 'ENOENT' + const err = new Error() + err.code = this.fakeCode + return this.Fs.open.callsArgWith(2, err, null) + }) - return it("should give a NotFoundError", function(done) { - return this.FSPersistorManager.getFileStream(this.location, this.name1, this.opts, (err,res)=> { - expect(res).to.equal(null); - expect(err).to.not.equal(null); - expect(err instanceof this.Errors.NotFoundError).to.equal(true); - return done(); - }); - }); - }); - - return describe("when some other error happens", function() { + return it('should give a NotFoundError', function(done) { + return this.FSPersistorManager.getFileStream( + this.location, + this.name1, + this.opts, + (err, res) => { + expect(res).to.equal(null) + expect(err).to.not.equal(null) + expect(err instanceof this.Errors.NotFoundError).to.equal(true) + return done() + } + ) + }) + }) + return describe('when some other error happens', function() { beforeEach(function() { - this.fakeCode = 'SOMETHINGHORRIBLE'; - const err = new Error(); - err.code = this.fakeCode; - return this.Fs.open.callsArgWith(2, err, null); - }); + this.fakeCode = 'SOMETHINGHORRIBLE' + const err = new Error() + err.code = this.fakeCode + return this.Fs.open.callsArgWith(2, err, null) + }) - return it("should give an Error", function(done) { - return this.FSPersistorManager.getFileStream(this.location, this.name1, this.opts, (err,res)=> { - expect(res).to.equal(null); - expect(err).to.not.equal(null); - expect(err instanceof Error).to.equal(true); - return done(); - }); - }); - }); - }); - }); + return it('should give an Error', function(done) { + return this.FSPersistorManager.getFileStream( + this.location, + this.name1, + this.opts, + (err, res) => { + expect(res).to.equal(null) + expect(err).to.not.equal(null) + expect(err instanceof Error).to.equal(true) + return done() + } + ) + }) + }) + }) + }) - describe("getFileSize", function() { - it("should return the file size", function(done) { - const expectedFileSize = 75382; - this.Fs.stat.yields(new Error("fs.stat got unexpected arguments")); - this.Fs.stat.withArgs(`${this.location}/${this.name1Filtered}`) - .yields(null, { size: expectedFileSize }); + describe('getFileSize', function() { + it('should return the file size', function(done) { + const expectedFileSize = 75382 + this.Fs.stat.yields(new Error('fs.stat got unexpected arguments')) + this.Fs.stat + .withArgs(`${this.location}/${this.name1Filtered}`) + .yields(null, { size: expectedFileSize }) - return this.FSPersistorManager.getFileSize(this.location, this.name1, (err, fileSize) => { - if (err != null) { - return done(err); + return this.FSPersistorManager.getFileSize( + this.location, + this.name1, + (err, fileSize) => { + if (err != null) { + return done(err) + } + expect(fileSize).to.equal(expectedFileSize) + return done() } - expect(fileSize).to.equal(expectedFileSize); - return done(); - }); - }); + ) + }) - it("should throw a NotFoundError if the file does not exist", function(done) { - const error = new Error(); - error.code = "ENOENT"; - this.Fs.stat.yields(error); + it('should throw a NotFoundError if the file does not exist', function(done) { + const error = new Error() + error.code = 'ENOENT' + this.Fs.stat.yields(error) - return this.FSPersistorManager.getFileSize(this.location, this.name1, (err, fileSize) => { - expect(err).to.be.instanceof(this.Errors.NotFoundError); - return done(); - }); - }); + return this.FSPersistorManager.getFileSize( + this.location, + this.name1, + (err, fileSize) => { + expect(err).to.be.instanceof(this.Errors.NotFoundError) + return done() + } + ) + }) - return it("should rethrow any other error", function(done) { - const error = new Error(); - this.Fs.stat.yields(error); + return it('should rethrow any other error', function(done) { + const error = new Error() + this.Fs.stat.yields(error) - return this.FSPersistorManager.getFileSize(this.location, this.name1, (err, fileSize) => { - expect(err).to.equal(error); - return done(); - }); - }); - }); + return this.FSPersistorManager.getFileSize( + this.location, + this.name1, + (err, fileSize) => { + expect(err).to.equal(error) + return done() + } + ) + }) + }) - describe("copyFile", function() { + describe('copyFile', function() { beforeEach(function() { - this.ReadStream= { + this.ReadStream = { on() {}, - pipe:sinon.stub() - }; - this.WriteStream= - {on() {}}; - this.Fs.createReadStream.returns(this.ReadStream); - return this.Fs.createWriteStream.returns(this.WriteStream); - }); + pipe: sinon.stub() + } + this.WriteStream = { on() {} } + this.Fs.createReadStream.returns(this.ReadStream) + return this.Fs.createWriteStream.returns(this.WriteStream) + }) - it("Should open the source for reading", function(done) { - this.FSPersistorManager.copyFile(this.location, this.name1, this.name2, function() {}); - this.Fs.createReadStream.calledWith(`${this.location}/${this.name1Filtered}`).should.equal(true); - return done(); - }); + it('Should open the source for reading', function(done) { + this.FSPersistorManager.copyFile( + this.location, + this.name1, + this.name2, + function() {} + ) + this.Fs.createReadStream + .calledWith(`${this.location}/${this.name1Filtered}`) + .should.equal(true) + return done() + }) - it("Should open the target for writing", function(done) { - this.FSPersistorManager.copyFile(this.location, this.name1, this.name2, function() {}); - this.Fs.createWriteStream.calledWith(`${this.location}/${this.name2}`).should.equal(true); - return done(); - }); + it('Should open the target for writing', function(done) { + this.FSPersistorManager.copyFile( + this.location, + this.name1, + this.name2, + function() {} + ) + this.Fs.createWriteStream + .calledWith(`${this.location}/${this.name2}`) + .should.equal(true) + return done() + }) - return it("Should pipe the source to the target", function(done) { - this.FSPersistorManager.copyFile(this.location, this.name1, this.name2, function() {}); - this.ReadStream.pipe.calledWith(this.WriteStream).should.equal(true); - return done(); - }); - }); + return it('Should pipe the source to the target', function(done) { + this.FSPersistorManager.copyFile( + this.location, + this.name1, + this.name2, + function() {} + ) + this.ReadStream.pipe.calledWith(this.WriteStream).should.equal(true) + return done() + }) + }) - describe("deleteFile", function() { + describe('deleteFile', function() { beforeEach(function() { - return this.Fs.unlink.callsArgWith(1,this.error); - }); + return this.Fs.unlink.callsArgWith(1, this.error) + }) - it("Should call unlink with correct options", function(done) { - return this.FSPersistorManager.deleteFile(this.location, this.name1, err => { - this.Fs.unlink.calledWith(`${this.location}/${this.name1Filtered}`).should.equal(true); - return done(); - }); - }); + it('Should call unlink with correct options', function(done) { + return this.FSPersistorManager.deleteFile( + this.location, + this.name1, + err => { + this.Fs.unlink + .calledWith(`${this.location}/${this.name1Filtered}`) + .should.equal(true) + return done() + } + ) + }) - return it("Should propogate the error", function(done) { - return this.FSPersistorManager.deleteFile(this.location, this.name1, err => { - err.should.equal(this.error); - return done(); - }); - }); - }); + return it('Should propogate the error', function(done) { + return this.FSPersistorManager.deleteFile( + this.location, + this.name1, + err => { + err.should.equal(this.error) + return done() + } + ) + }) + }) - - describe("deleteDirectory", function() { + describe('deleteDirectory', function() { beforeEach(function() { - return this.Rimraf.callsArgWith(1,this.error); - }); + return this.Rimraf.callsArgWith(1, this.error) + }) - it("Should call rmdir(rimraf) with correct options", function(done) { - return this.FSPersistorManager.deleteDirectory(this.location, this.name1, err => { - this.Rimraf.calledWith(`${this.location}/${this.name1Filtered}`).should.equal(true); - return done(); - }); - }); + it('Should call rmdir(rimraf) with correct options', function(done) { + return this.FSPersistorManager.deleteDirectory( + this.location, + this.name1, + err => { + this.Rimraf.calledWith( + `${this.location}/${this.name1Filtered}` + ).should.equal(true) + return done() + } + ) + }) - return it("Should propogate the error", function(done) { - return this.FSPersistorManager.deleteDirectory(this.location, this.name1, err => { - err.should.equal(this.error); - return done(); - }); - }); - }); + return it('Should propogate the error', function(done) { + return this.FSPersistorManager.deleteDirectory( + this.location, + this.name1, + err => { + err.should.equal(this.error) + return done() + } + ) + }) + }) - describe("checkIfFileExists", function() { + describe('checkIfFileExists', function() { beforeEach(function() { - return this.Fs.exists.callsArgWith(1,true); - }); + return this.Fs.exists.callsArgWith(1, true) + }) - it("Should call exists with correct options", function(done) { - return this.FSPersistorManager.checkIfFileExists(this.location, this.name1, exists => { - this.Fs.exists.calledWith(`${this.location}/${this.name1Filtered}`).should.equal(true); - return done(); - }); - }); + it('Should call exists with correct options', function(done) { + return this.FSPersistorManager.checkIfFileExists( + this.location, + this.name1, + exists => { + this.Fs.exists + .calledWith(`${this.location}/${this.name1Filtered}`) + .should.equal(true) + return done() + } + ) + }) // fs.exists simply returns false on any error, so... - it("should not return an error", function(done) { - return this.FSPersistorManager.checkIfFileExists(this.location, this.name1, (err,exists) => { - expect(err).to.be.null; - return done(); - }); - }); + it('should not return an error', function(done) { + return this.FSPersistorManager.checkIfFileExists( + this.location, + this.name1, + (err, exists) => { + expect(err).to.be.null + return done() + } + ) + }) - it("Should return true for existing files", function(done) { - this.Fs.exists.callsArgWith(1,true); - return this.FSPersistorManager.checkIfFileExists(this.location, this.name1, (err,exists) => { - exists.should.be.true; - return done(); - }); - }); + it('Should return true for existing files', function(done) { + this.Fs.exists.callsArgWith(1, true) + return this.FSPersistorManager.checkIfFileExists( + this.location, + this.name1, + (err, exists) => { + exists.should.be.true + return done() + } + ) + }) - return it("Should return false for non-existing files", function(done) { - this.Fs.exists.callsArgWith(1,false); - return this.FSPersistorManager.checkIfFileExists(this.location, this.name1, (err,exists) => { - exists.should.be.false; - return done(); - }); - }); - }); + return it('Should return false for non-existing files', function(done) { + this.Fs.exists.callsArgWith(1, false) + return this.FSPersistorManager.checkIfFileExists( + this.location, + this.name1, + (err, exists) => { + exists.should.be.false + return done() + } + ) + }) + }) - return describe("directorySize", function() { + return describe('directorySize', function() { + it('should propogate the error', function(done) { + this.Fs.readdir.callsArgWith(1, this.error) + return this.FSPersistorManager.directorySize( + this.location, + this.name1, + (err, totalsize) => { + err.should.equal(this.error) + return done() + } + ) + }) - it("should propogate the error", function(done) { - this.Fs.readdir.callsArgWith(1, this.error); - return this.FSPersistorManager.directorySize(this.location, this.name1, (err, totalsize) => { - err.should.equal(this.error); - return done(); - }); - }); - - return it("should sum directory files size", function(done) { - this.Fs.readdir.callsArgWith(1, null, [ {'file1': 'file1'}, {'file2': 'file2'} ]); - this.Fs.fstatSync.returns({size : 1024}); - return this.FSPersistorManager.directorySize(this.location, this.name1, (err, totalsize) => { - expect(totalsize).to.equal(2048); - return done(); - }); - }); - }); -}); + return it('should sum directory files size', function(done) { + this.Fs.readdir.callsArgWith(1, null, [ + { file1: 'file1' }, + { file2: 'file2' } + ]) + this.Fs.fstatSync.returns({ size: 1024 }) + return this.FSPersistorManager.directorySize( + this.location, + this.name1, + (err, totalsize) => { + expect(totalsize).to.equal(2048) + return done() + } + ) + }) + }) +}) diff --git a/services/filestore/test/unit/js/FileControllerTests.js b/services/filestore/test/unit/js/FileControllerTests.js index d60ca5fbd1..2505e87c9b 100644 --- a/services/filestore/test/unit/js/FileControllerTests.js +++ b/services/filestore/test/unit/js/FileControllerTests.js @@ -9,281 +9,289 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const { - assert -} = require("chai"); -const sinon = require('sinon'); -const chai = require('chai'); -const should = chai.should(); -const { - expect -} = chai; -const modulePath = "../../../app/js/FileController.js"; -const SandboxedModule = require('sandboxed-module'); +const { assert } = require('chai') +const sinon = require('sinon') +const chai = require('chai') +const should = chai.should() +const { expect } = chai +const modulePath = '../../../app/js/FileController.js' +const SandboxedModule = require('sandboxed-module') -describe("FileController", function() { +describe('FileController', function() { + beforeEach(function() { + this.PersistorManager = { + sendStream: sinon.stub(), + copyFile: sinon.stub(), + deleteFile: sinon.stub() + } - beforeEach(function() { - this.PersistorManager = { - sendStream: sinon.stub(), - copyFile: sinon.stub(), - deleteFile:sinon.stub() - }; + this.settings = { + s3: { + buckets: { + user_files: 'user_files' + } + } + } + this.FileHandler = { + getFile: sinon.stub(), + getFileSize: sinon.stub(), + deleteFile: sinon.stub(), + insertFile: sinon.stub(), + getDirectorySize: sinon.stub() + } + this.LocalFileWriter = {} + this.controller = SandboxedModule.require(modulePath, { + requires: { + './LocalFileWriter': this.LocalFileWriter, + './FileHandler': this.FileHandler, + './PersistorManager': this.PersistorManager, + './Errors': (this.Errors = { NotFoundError: sinon.stub() }), + 'settings-sharelatex': this.settings, + 'metrics-sharelatex': { + inc() {} + }, + 'logger-sharelatex': { + log() {}, + err() {} + } + } + }) + this.project_id = 'project_id' + this.file_id = 'file_id' + this.bucket = 'user_files' + this.key = `${this.project_id}/${this.file_id}` + this.req = { + key: this.key, + bucket: this.bucket, + query: {}, + params: { + project_id: this.project_id, + file_id: this.file_id + }, + headers: {} + } + this.res = { + set: sinon.stub().returnsThis(), + status: sinon.stub().returnsThis() + } + return (this.fileStream = {}) + }) - this.settings = { - s3: { - buckets: { - user_files:"user_files" - } - } - }; - this.FileHandler = { - getFile: sinon.stub(), - getFileSize: sinon.stub(), - deleteFile: sinon.stub(), - insertFile: sinon.stub(), - getDirectorySize: sinon.stub() - }; - this.LocalFileWriter = {}; - this.controller = SandboxedModule.require(modulePath, { requires: { - "./LocalFileWriter":this.LocalFileWriter, - "./FileHandler": this.FileHandler, - "./PersistorManager":this.PersistorManager, - "./Errors": (this.Errors = - {NotFoundError: sinon.stub()}), - "settings-sharelatex": this.settings, - "metrics-sharelatex": { - inc() {} - }, - "logger-sharelatex": { - log() {}, - err() {} - } - } - } - ); - this.project_id = "project_id"; - this.file_id = "file_id"; - this.bucket = "user_files"; - this.key = `${this.project_id}/${this.file_id}`; - this.req = { - key:this.key, - bucket:this.bucket, - query:{}, - params: { - project_id:this.project_id, - file_id:this.file_id - }, - headers: {} - }; - this.res = { - set: sinon.stub().returnsThis(), - status: sinon.stub().returnsThis() - }; - return this.fileStream = {};}); + describe('getFile', function() { + it('should pipe the stream', function(done) { + this.FileHandler.getFile.callsArgWith(3, null, this.fileStream) + this.fileStream.pipe = res => { + res.should.equal(this.res) + return done() + } + return this.controller.getFile(this.req, this.res) + }) - describe("getFile", function() { + it('should send a 200 if the cacheWarm param is true', function(done) { + this.req.query.cacheWarm = true + this.FileHandler.getFile.callsArgWith(3, null, this.fileStream) + this.res.send = statusCode => { + statusCode.should.equal(200) + return done() + } + return this.controller.getFile(this.req, this.res) + }) - it("should pipe the stream", function(done){ - this.FileHandler.getFile.callsArgWith(3, null, this.fileStream); - this.fileStream.pipe = res=> { - res.should.equal(this.res); - return done(); - }; - return this.controller.getFile(this.req, this.res); - }); + it('should send a 500 if there is a problem', function(done) { + this.FileHandler.getFile.callsArgWith(3, 'error') + this.res.send = code => { + code.should.equal(500) + return done() + } + return this.controller.getFile(this.req, this.res) + }) - it("should send a 200 if the cacheWarm param is true", function(done){ - this.req.query.cacheWarm = true; - this.FileHandler.getFile.callsArgWith(3, null, this.fileStream); - this.res.send = statusCode=> { - statusCode.should.equal(200); - return done(); - }; - return this.controller.getFile(this.req, this.res); - }); + return describe("with a 'Range' header set", function() { + beforeEach(function() { + return (this.req.headers.range = 'bytes=0-8') + }) - it("should send a 500 if there is a problem", function(done){ - this.FileHandler.getFile.callsArgWith(3, "error"); - this.res.send = code=> { - code.should.equal(500); - return done(); - }; - return this.controller.getFile(this.req, this.res); - }); + return it("should pass 'start' and 'end' options to FileHandler", function(done) { + this.FileHandler.getFile.callsArgWith(3, null, this.fileStream) + this.fileStream.pipe = res => { + expect(this.FileHandler.getFile.lastCall.args[2].start).to.equal(0) + expect(this.FileHandler.getFile.lastCall.args[2].end).to.equal(8) + return done() + } + return this.controller.getFile(this.req, this.res) + }) + }) + }) - return describe("with a 'Range' header set", function() { + describe('getFileHead', function() { + it('should return the file size in a Content-Length header', function(done) { + const expectedFileSize = 84921 + this.FileHandler.getFileSize.yields( + new Error('FileHandler.getFileSize: unexpected arguments') + ) + this.FileHandler.getFileSize + .withArgs(this.bucket, this.key) + .yields(null, expectedFileSize) - beforeEach(function() { - return this.req.headers.range = 'bytes=0-8'; - }); + this.res.end = () => { + expect(this.res.status.lastCall.args[0]).to.equal(200) + expect( + this.res.set.calledWith('Content-Length', expectedFileSize) + ).to.equal(true) + return done() + } - return it("should pass 'start' and 'end' options to FileHandler", function(done) { - this.FileHandler.getFile.callsArgWith(3, null, this.fileStream); - this.fileStream.pipe = res=> { - expect(this.FileHandler.getFile.lastCall.args[2].start).to.equal(0); - expect(this.FileHandler.getFile.lastCall.args[2].end).to.equal(8); - return done(); - }; - return this.controller.getFile(this.req, this.res); - }); - }); - }); + return this.controller.getFileHead(this.req, this.res) + }) - describe("getFileHead", function() { - it("should return the file size in a Content-Length header", function(done) { - const expectedFileSize = 84921; - this.FileHandler.getFileSize.yields( - new Error("FileHandler.getFileSize: unexpected arguments") - ); - this.FileHandler.getFileSize.withArgs(this.bucket, this.key).yields(null, expectedFileSize); + it('should return a 404 is the file is not found', function(done) { + this.FileHandler.getFileSize.yields(new this.Errors.NotFoundError()) - this.res.end = () => { - expect(this.res.status.lastCall.args[0]).to.equal(200); - expect(this.res.set.calledWith("Content-Length", expectedFileSize)).to.equal(true); - return done(); - }; + this.res.end = () => { + expect(this.res.status.lastCall.args[0]).to.equal(404) + return done() + } - return this.controller.getFileHead(this.req, this.res); - }); + return this.controller.getFileHead(this.req, this.res) + }) - it("should return a 404 is the file is not found", function(done) { - this.FileHandler.getFileSize.yields(new this.Errors.NotFoundError()); + return it('should return a 500 on internal errors', function(done) { + this.FileHandler.getFileSize.yields(new Error()) - this.res.end = () => { - expect(this.res.status.lastCall.args[0]).to.equal(404); - return done(); - }; + this.res.end = () => { + expect(this.res.status.lastCall.args[0]).to.equal(500) + return done() + } - return this.controller.getFileHead(this.req, this.res); - }); + return this.controller.getFileHead(this.req, this.res) + }) + }) - return it("should return a 500 on internal errors", function(done) { - this.FileHandler.getFileSize.yields(new Error()); + describe('insertFile', () => + it('should send bucket name key and res to PersistorManager', function(done) { + this.FileHandler.insertFile.callsArgWith(3) + this.res.send = () => { + this.FileHandler.insertFile + .calledWith(this.bucket, this.key, this.req) + .should.equal(true) + return done() + } + return this.controller.insertFile(this.req, this.res) + })) - this.res.end = () => { - expect(this.res.status.lastCall.args[0]).to.equal(500); - return done(); - }; + describe('copyFile', function() { + beforeEach(function() { + this.oldFile_id = 'old_file_id' + this.oldProject_id = 'old_project_id' + return (this.req.body = { + source: { + project_id: this.oldProject_id, + file_id: this.oldFile_id + } + }) + }) - return this.controller.getFileHead(this.req, this.res); - }); - }); + it('should send bucket name and both keys to PersistorManager', function(done) { + this.PersistorManager.copyFile.callsArgWith(3) + this.res.send = code => { + code.should.equal(200) + this.PersistorManager.copyFile + .calledWith( + this.bucket, + `${this.oldProject_id}/${this.oldFile_id}`, + this.key + ) + .should.equal(true) + return done() + } + return this.controller.copyFile(this.req, this.res) + }) - describe("insertFile", () => it("should send bucket name key and res to PersistorManager", function(done){ - this.FileHandler.insertFile.callsArgWith(3); - this.res.send = () => { - this.FileHandler.insertFile.calledWith(this.bucket, this.key, this.req).should.equal(true); - return done(); - }; - return this.controller.insertFile(this.req, this.res); - })); + it('should send a 404 if the original file was not found', function(done) { + this.PersistorManager.copyFile.callsArgWith( + 3, + new this.Errors.NotFoundError() + ) + this.res.send = code => { + code.should.equal(404) + return done() + } + return this.controller.copyFile(this.req, this.res) + }) - describe("copyFile", function() { - beforeEach(function() { - this.oldFile_id = "old_file_id"; - this.oldProject_id = "old_project_id"; - return this.req.body = { - source: { - project_id: this.oldProject_id, - file_id: this.oldFile_id - } - }; - }); + return it('should send a 500 if there was an error', function(done) { + this.PersistorManager.copyFile.callsArgWith(3, 'error') + this.res.send = code => { + code.should.equal(500) + return done() + } + return this.controller.copyFile(this.req, this.res) + }) + }) - it("should send bucket name and both keys to PersistorManager", function(done){ - this.PersistorManager.copyFile.callsArgWith(3); - this.res.send = code=> { - code.should.equal(200); - this.PersistorManager.copyFile.calledWith(this.bucket, `${this.oldProject_id}/${this.oldFile_id}`, this.key).should.equal(true); - return done(); - }; - return this.controller.copyFile(this.req, this.res); - }); + describe('delete file', function() { + it('should tell the file handler', function(done) { + this.FileHandler.deleteFile.callsArgWith(2) + this.res.send = code => { + code.should.equal(204) + this.FileHandler.deleteFile + .calledWith(this.bucket, this.key) + .should.equal(true) + return done() + } + return this.controller.deleteFile(this.req, this.res) + }) - it("should send a 404 if the original file was not found", function(done) { - this.PersistorManager.copyFile.callsArgWith(3, new this.Errors.NotFoundError()); - this.res.send = code=> { - code.should.equal(404); - return done(); - }; - return this.controller.copyFile(this.req, this.res); - }); + return it('should send a 500 if there was an error', function(done) { + this.FileHandler.deleteFile.callsArgWith(2, 'error') + this.res.send = function(code) { + code.should.equal(500) + return done() + } + return this.controller.deleteFile(this.req, this.res) + }) + }) - return it("should send a 500 if there was an error", function(done){ - this.PersistorManager.copyFile.callsArgWith(3, "error"); - this.res.send = code=> { - code.should.equal(500); - return done(); - }; - return this.controller.copyFile(this.req, this.res); - }); - }); + describe('_get_range', function() { + it('should parse a valid Range header', function(done) { + const result = this.controller._get_range('bytes=0-200') + expect(result).to.not.equal(null) + expect(result.start).to.equal(0) + expect(result.end).to.equal(200) + return done() + }) - describe("delete file", function() { + it('should return null for an invalid Range header', function(done) { + const result = this.controller._get_range('wat') + expect(result).to.equal(null) + return done() + }) - it("should tell the file handler", function(done){ - this.FileHandler.deleteFile.callsArgWith(2); - this.res.send = code=> { - code.should.equal(204); - this.FileHandler.deleteFile.calledWith(this.bucket, this.key).should.equal(true); - return done(); - }; - return this.controller.deleteFile(this.req, this.res); - }); + return it("should return null for any type other than 'bytes'", function(done) { + const result = this.controller._get_range('carrots=0-200') + expect(result).to.equal(null) + return done() + }) + }) - return it("should send a 500 if there was an error", function(done){ - this.FileHandler.deleteFile.callsArgWith(2, "error"); - this.res.send = function(code){ - code.should.equal(500); - return done(); - }; - return this.controller.deleteFile(this.req, this.res); - }); - }); + return describe('directorySize', function() { + it('should return total directory size bytes', function(done) { + this.FileHandler.getDirectorySize.callsArgWith(2, null, 1024) + return this.controller.directorySize(this.req, { + json: result => { + expect(result['total bytes']).to.equal(1024) + return done() + } + }) + }) - describe("_get_range", function() { - - it("should parse a valid Range header", function(done) { - const result = this.controller._get_range('bytes=0-200'); - expect(result).to.not.equal(null); - expect(result.start).to.equal(0); - expect(result.end).to.equal(200); - return done(); - }); - - it("should return null for an invalid Range header", function(done) { - const result = this.controller._get_range('wat'); - expect(result).to.equal(null); - return done(); - }); - - return it("should return null for any type other than 'bytes'", function(done) { - const result = this.controller._get_range('carrots=0-200'); - expect(result).to.equal(null); - return done(); - }); - }); - - return describe("directorySize", function() { - - it("should return total directory size bytes", function(done) { - this.FileHandler.getDirectorySize.callsArgWith(2, null, 1024); - return this.controller.directorySize(this.req, { json:result=> { - expect(result['total bytes']).to.equal(1024); - return done(); - } - } - ); - }); - - return it("should send a 500 if there was an error", function(done){ - this.FileHandler.getDirectorySize.callsArgWith(2, "error"); - this.res.send = function(code){ - code.should.equal(500); - return done(); - }; - return this.controller.directorySize(this.req, this.res); - }); - }); -}); + return it('should send a 500 if there was an error', function(done) { + this.FileHandler.getDirectorySize.callsArgWith(2, 'error') + this.res.send = function(code) { + code.should.equal(500) + return done() + } + return this.controller.directorySize(this.req, this.res) + }) + }) +}) diff --git a/services/filestore/test/unit/js/FileConverterTests.js b/services/filestore/test/unit/js/FileConverterTests.js index 96cb0ee53a..1b548d592e 100644 --- a/services/filestore/test/unit/js/FileConverterTests.js +++ b/services/filestore/test/unit/js/FileConverterTests.js @@ -10,110 +10,110 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const { - assert -} = require("chai"); -const sinon = require('sinon'); -const chai = require('chai'); -const should = chai.should(); -const { - expect -} = chai; -const modulePath = "../../../app/js/FileConverter.js"; -const SandboxedModule = require('sandboxed-module'); +const { assert } = require('chai') +const sinon = require('sinon') +const chai = require('chai') +const should = chai.should() +const { expect } = chai +const modulePath = '../../../app/js/FileConverter.js' +const SandboxedModule = require('sandboxed-module') -describe("FileConverter", function() { +describe('FileConverter', function() { + beforeEach(function() { + this.safe_exec = sinon.stub() + this.converter = SandboxedModule.require(modulePath, { + requires: { + './SafeExec': this.safe_exec, + 'logger-sharelatex': { + log() {}, + err() {} + }, + 'metrics-sharelatex': { + inc() {}, + Timer() { + return { done() {} } + } + }, + 'settings-sharelatex': (this.Settings = { + commands: { + convertCommandPrefix: [] + } + }) + } + }) - beforeEach(function() { + this.sourcePath = '/this/path/here.eps' + this.format = 'png' + return (this.error = 'Error') + }) - this.safe_exec = sinon.stub(); - this.converter = SandboxedModule.require(modulePath, { requires: { - "./SafeExec": this.safe_exec, - "logger-sharelatex": { - log() {}, - err() {} - }, - "metrics-sharelatex": { - inc() {}, - Timer() { - return {done() {}}; - } - }, - "settings-sharelatex": (this.Settings = { - commands: { - convertCommandPrefix: [] - } - }) - } - }); + describe('convert', function() { + it('should convert the source to the requested format', function(done) { + this.safe_exec.callsArgWith(2) + return this.converter.convert(this.sourcePath, this.format, err => { + const args = this.safe_exec.args[0][0] + args.indexOf(`${this.sourcePath}[0]`).should.not.equal(-1) + args.indexOf(`${this.sourcePath}.${this.format}`).should.not.equal(-1) + return done() + }) + }) - this.sourcePath = "/this/path/here.eps"; - this.format = "png"; - return this.error = "Error"; - }); + it('should return the dest path', function(done) { + this.safe_exec.callsArgWith(2) + return this.converter.convert( + this.sourcePath, + this.format, + (err, destPath) => { + destPath.should.equal(`${this.sourcePath}.${this.format}`) + return done() + } + ) + }) - describe("convert", function() { + it('should return the error from convert', function(done) { + this.safe_exec.callsArgWith(2, this.error) + return this.converter.convert(this.sourcePath, this.format, err => { + err.should.equal(this.error) + return done() + }) + }) - it("should convert the source to the requested format", function(done){ - this.safe_exec.callsArgWith(2); - return this.converter.convert(this.sourcePath, this.format, err=> { - const args = this.safe_exec.args[0][0]; - args.indexOf(`${this.sourcePath}[0]`).should.not.equal(-1); - args.indexOf(`${this.sourcePath}.${this.format}`).should.not.equal(-1); - return done(); - }); - }); + it('should not accapt an non aproved format', function(done) { + this.safe_exec.callsArgWith(2) + return this.converter.convert(this.sourcePath, 'ahhhhh', err => { + expect(err).to.exist + return done() + }) + }) - it("should return the dest path", function(done){ - this.safe_exec.callsArgWith(2); - return this.converter.convert(this.sourcePath, this.format, (err, destPath)=> { - destPath.should.equal(`${this.sourcePath}.${this.format}`); - return done(); - }); - }); + return it('should prefix the command with Settings.commands.convertCommandPrefix', function(done) { + this.safe_exec.callsArgWith(2) + this.Settings.commands.convertCommandPrefix = ['nice'] + return this.converter.convert(this.sourcePath, this.format, err => { + const command = this.safe_exec.args[0][0] + command[0].should.equal('nice') + return done() + }) + }) + }) - it("should return the error from convert", function(done){ - this.safe_exec.callsArgWith(2, this.error); - return this.converter.convert(this.sourcePath, this.format, err=> { - err.should.equal(this.error); - return done(); - }); - }); + describe('thumbnail', () => + it('should call converter resize with args', function(done) { + this.safe_exec.callsArgWith(2) + return this.converter.thumbnail(this.sourcePath, err => { + const args = this.safe_exec.args[0][0] + args.indexOf(`${this.sourcePath}[0]`).should.not.equal(-1) + return done() + }) + })) - it("should not accapt an non aproved format", function(done){ - this.safe_exec.callsArgWith(2); - return this.converter.convert(this.sourcePath, "ahhhhh", err=> { - expect(err).to.exist; - return done(); - }); - }); - - return it("should prefix the command with Settings.commands.convertCommandPrefix", function(done) { - this.safe_exec.callsArgWith(2); - this.Settings.commands.convertCommandPrefix = ["nice"]; - return this.converter.convert(this.sourcePath, this.format, err=> { - const command = this.safe_exec.args[0][0]; - command[0].should.equal("nice"); - return done(); - }); - }); - }); - - describe("thumbnail", () => it("should call converter resize with args", function(done){ - this.safe_exec.callsArgWith(2); - return this.converter.thumbnail(this.sourcePath, err=> { - const args = this.safe_exec.args[0][0]; - args.indexOf(`${this.sourcePath}[0]`).should.not.equal(-1); - return done(); - }); - })); - - return describe("preview", () => it("should call converter resize with args", function(done){ - this.safe_exec.callsArgWith(2); - return this.converter.preview(this.sourcePath, err=> { - const args = this.safe_exec.args[0][0]; - args.indexOf(`${this.sourcePath}[0]`).should.not.equal(-1); - return done(); - }); - })); -}); + return describe('preview', () => + it('should call converter resize with args', function(done) { + this.safe_exec.callsArgWith(2) + return this.converter.preview(this.sourcePath, err => { + const args = this.safe_exec.args[0][0] + args.indexOf(`${this.sourcePath}[0]`).should.not.equal(-1) + return done() + }) + })) +}) diff --git a/services/filestore/test/unit/js/FileHandlerTests.js b/services/filestore/test/unit/js/FileHandlerTests.js index 13c60f08da..e641ffdd16 100644 --- a/services/filestore/test/unit/js/FileHandlerTests.js +++ b/services/filestore/test/unit/js/FileHandlerTests.js @@ -10,273 +10,359 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const { - assert -} = require("chai"); -const sinon = require('sinon'); -const chai = require('chai'); -const should = chai.should(); -const { - expect -} = chai; -const modulePath = "../../../app/js/FileHandler.js"; -const SandboxedModule = require('sandboxed-module'); +const { assert } = require('chai') +const sinon = require('sinon') +const chai = require('chai') +const should = chai.should() +const { expect } = chai +const modulePath = '../../../app/js/FileHandler.js' +const SandboxedModule = require('sandboxed-module') -describe("FileHandler", function() { +describe('FileHandler', function() { + beforeEach(function() { + this.settings = { + s3: { + buckets: { + user_files: 'user_files' + } + } + } + this.PersistorManager = { + getFileStream: sinon.stub(), + checkIfFileExists: sinon.stub(), + deleteFile: sinon.stub(), + deleteDirectory: sinon.stub(), + sendStream: sinon.stub(), + insertFile: sinon.stub(), + directorySize: sinon.stub() + } + this.LocalFileWriter = { + writeStream: sinon.stub(), + getStream: sinon.stub(), + deleteFile: sinon.stub() + } + this.FileConverter = { + convert: sinon.stub(), + thumbnail: sinon.stub(), + preview: sinon.stub() + } + this.keyBuilder = { + addCachingToKey: sinon.stub(), + getConvertedFolderKey: sinon.stub() + } + this.ImageOptimiser = { compressPng: sinon.stub() } + this.handler = SandboxedModule.require(modulePath, { + requires: { + 'settings-sharelatex': this.settings, + './PersistorManager': this.PersistorManager, + './LocalFileWriter': this.LocalFileWriter, + './FileConverter': this.FileConverter, + './KeyBuilder': this.keyBuilder, + './ImageOptimiser': this.ImageOptimiser, + 'logger-sharelatex': { + log() {}, + err() {} + } + } + }) + this.bucket = 'my_bucket' + this.key = 'key/here' + this.stubbedPath = '/var/somewhere/path' + this.format = 'png' + return (this.formattedStubbedPath = `${this.stubbedPath}.${this.format}`) + }) - beforeEach(function() { - this.settings = { - s3: { - buckets: { - user_files:"user_files" - } - } - }; - this.PersistorManager = { - getFileStream: sinon.stub(), - checkIfFileExists: sinon.stub(), - deleteFile: sinon.stub(), - deleteDirectory: sinon.stub(), - sendStream: sinon.stub(), - insertFile: sinon.stub(), - directorySize: sinon.stub() - }; - this.LocalFileWriter = { - writeStream: sinon.stub(), - getStream: sinon.stub(), - deleteFile: sinon.stub() - }; - this.FileConverter = { - convert: sinon.stub(), - thumbnail: sinon.stub(), - preview: sinon.stub() - }; - this.keyBuilder = { - addCachingToKey: sinon.stub(), - getConvertedFolderKey: sinon.stub() - }; - this.ImageOptimiser = - {compressPng: sinon.stub()}; - this.handler = SandboxedModule.require(modulePath, { requires: { - "settings-sharelatex": this.settings, - "./PersistorManager":this.PersistorManager, - "./LocalFileWriter":this.LocalFileWriter, - "./FileConverter":this.FileConverter, - "./KeyBuilder": this.keyBuilder, - "./ImageOptimiser":this.ImageOptimiser, - "logger-sharelatex": { - log() {}, - err() {} - } - } - } - ); - this.bucket = "my_bucket"; - this.key = "key/here"; - this.stubbedPath = "/var/somewhere/path"; - this.format = "png"; - return this.formattedStubbedPath = `${this.stubbedPath}.${this.format}`; - }); + describe('insertFile', function() { + beforeEach(function() { + this.stream = {} + this.PersistorManager.deleteDirectory.callsArgWith(2) + return this.PersistorManager.sendStream.callsArgWith(3) + }) - describe("insertFile", function() { - beforeEach(function() { - this.stream = {}; - this.PersistorManager.deleteDirectory.callsArgWith(2); - return this.PersistorManager.sendStream.callsArgWith(3); - }); + it('should send file to the filestore', function(done) { + return this.handler.insertFile(this.bucket, this.key, this.stream, () => { + this.PersistorManager.sendStream + .calledWith(this.bucket, this.key, this.stream) + .should.equal(true) + return done() + }) + }) - it("should send file to the filestore", function(done){ - return this.handler.insertFile(this.bucket, this.key, this.stream, () => { - this.PersistorManager.sendStream.calledWith(this.bucket, this.key, this.stream).should.equal(true); - return done(); - }); - }); + return it('should delete the convetedKey folder', function(done) { + this.keyBuilder.getConvertedFolderKey.returns(this.stubbedConvetedKey) + return this.handler.insertFile(this.bucket, this.key, this.stream, () => { + this.PersistorManager.deleteDirectory + .calledWith(this.bucket, this.stubbedConvetedKey) + .should.equal(true) + return done() + }) + }) + }) - return it("should delete the convetedKey folder", function(done){ - this.keyBuilder.getConvertedFolderKey.returns(this.stubbedConvetedKey); - return this.handler.insertFile(this.bucket, this.key, this.stream, () => { - this.PersistorManager.deleteDirectory.calledWith(this.bucket, this.stubbedConvetedKey).should.equal(true); - return done(); - }); - }); - }); + describe('deleteFile', function() { + beforeEach(function() { + this.keyBuilder.getConvertedFolderKey.returns(this.stubbedConvetedKey) + this.PersistorManager.deleteFile.callsArgWith(2) + return this.PersistorManager.deleteDirectory.callsArgWith(2) + }) - describe("deleteFile", function() { - beforeEach(function() { - this.keyBuilder.getConvertedFolderKey.returns(this.stubbedConvetedKey); - this.PersistorManager.deleteFile.callsArgWith(2); - return this.PersistorManager.deleteDirectory.callsArgWith(2); - }); + it('should tell the filestore manager to delete the file', function(done) { + return this.handler.deleteFile(this.bucket, this.key, () => { + this.PersistorManager.deleteFile + .calledWith(this.bucket, this.key) + .should.equal(true) + return done() + }) + }) - it("should tell the filestore manager to delete the file", function(done){ - return this.handler.deleteFile(this.bucket, this.key, () => { - this.PersistorManager.deleteFile.calledWith(this.bucket, this.key).should.equal(true); - return done(); - }); - }); + return it('should tell the filestore manager to delete the cached foler', function(done) { + return this.handler.deleteFile(this.bucket, this.key, () => { + this.PersistorManager.deleteDirectory + .calledWith(this.bucket, this.stubbedConvetedKey) + .should.equal(true) + return done() + }) + }) + }) - return it("should tell the filestore manager to delete the cached foler", function(done){ - return this.handler.deleteFile(this.bucket, this.key, () => { - this.PersistorManager.deleteDirectory.calledWith(this.bucket, this.stubbedConvetedKey).should.equal(true); - return done(); - }); - }); - }); + describe('getFile', function() { + beforeEach(function() { + this.handler._getStandardFile = sinon.stub().callsArgWith(3) + return (this.handler._getConvertedFile = sinon.stub().callsArgWith(3)) + }) - describe("getFile", function() { - beforeEach(function() { - this.handler._getStandardFile = sinon.stub().callsArgWith(3); - return this.handler._getConvertedFile = sinon.stub().callsArgWith(3); - }); + it('should call _getStandardFile if no format or style are defined', function(done) { + return this.handler.getFile(this.bucket, this.key, null, () => { + this.handler._getStandardFile.called.should.equal(true) + this.handler._getConvertedFile.called.should.equal(false) + return done() + }) + }) - it("should call _getStandardFile if no format or style are defined", function(done){ + it('should pass options to _getStandardFile', function(done) { + const options = { start: 0, end: 8 } + return this.handler.getFile(this.bucket, this.key, options, () => { + expect(this.handler._getStandardFile.lastCall.args[2].start).to.equal(0) + expect(this.handler._getStandardFile.lastCall.args[2].end).to.equal(8) + return done() + }) + }) - return this.handler.getFile(this.bucket, this.key, null, () => { - this.handler._getStandardFile.called.should.equal(true); - this.handler._getConvertedFile.called.should.equal(false); - return done(); - }); - }); + return it('should call _getConvertedFile if a format is defined', function(done) { + return this.handler.getFile( + this.bucket, + this.key, + { format: 'png' }, + () => { + this.handler._getStandardFile.called.should.equal(false) + this.handler._getConvertedFile.called.should.equal(true) + return done() + } + ) + }) + }) - it("should pass options to _getStandardFile", function(done) { - const options = {start: 0, end: 8}; - return this.handler.getFile(this.bucket, this.key, options, () => { - expect(this.handler._getStandardFile.lastCall.args[2].start).to.equal(0); - expect(this.handler._getStandardFile.lastCall.args[2].end).to.equal(8); - return done(); - }); - }); + describe('_getStandardFile', function() { + beforeEach(function() { + this.fileStream = { on() {} } + return this.PersistorManager.getFileStream.callsArgWith( + 3, + 'err', + this.fileStream + ) + }) - return it("should call _getConvertedFile if a format is defined", function(done){ - return this.handler.getFile(this.bucket, this.key, {format:"png"}, () => { - this.handler._getStandardFile.called.should.equal(false); - this.handler._getConvertedFile.called.should.equal(true); - return done(); - }); - }); - }); + it('should get the stream', function(done) { + return this.handler.getFile(this.bucket, this.key, null, () => { + this.PersistorManager.getFileStream + .calledWith(this.bucket, this.key) + .should.equal(true) + return done() + }) + }) - describe("_getStandardFile", function() { + it('should return the stream and error', function(done) { + return this.handler.getFile( + this.bucket, + this.key, + null, + (err, stream) => { + err.should.equal('err') + stream.should.equal(this.fileStream) + return done() + } + ) + }) - beforeEach(function() { - this.fileStream = {on() {}}; - return this.PersistorManager.getFileStream.callsArgWith(3, "err", this.fileStream); - }); + return it('should pass options to PersistorManager', function(done) { + return this.handler.getFile( + this.bucket, + this.key, + { start: 0, end: 8 }, + () => { + expect( + this.PersistorManager.getFileStream.lastCall.args[2].start + ).to.equal(0) + expect( + this.PersistorManager.getFileStream.lastCall.args[2].end + ).to.equal(8) + return done() + } + ) + }) + }) - it("should get the stream", function(done){ - return this.handler.getFile(this.bucket, this.key, null, () => { - this.PersistorManager.getFileStream.calledWith(this.bucket, this.key).should.equal(true); - return done(); - }); - }); + describe('_getConvertedFile', function() { + it('should getFileStream if it does exists', function(done) { + this.PersistorManager.checkIfFileExists.callsArgWith(2, null, true) + this.PersistorManager.getFileStream.callsArgWith(3) + return this.handler._getConvertedFile(this.bucket, this.key, {}, () => { + this.PersistorManager.getFileStream + .calledWith(this.bucket) + .should.equal(true) + return done() + }) + }) - it("should return the stream and error", function(done){ - return this.handler.getFile(this.bucket, this.key, null, (err, stream)=> { - err.should.equal("err"); - stream.should.equal(this.fileStream); - return done(); - }); - }); + return it('should call _getConvertedFileAndCache if it does exists', function(done) { + this.PersistorManager.checkIfFileExists.callsArgWith(2, null, false) + this.handler._getConvertedFileAndCache = sinon.stub().callsArgWith(4) + return this.handler._getConvertedFile(this.bucket, this.key, {}, () => { + this.handler._getConvertedFileAndCache + .calledWith(this.bucket, this.key) + .should.equal(true) + return done() + }) + }) + }) - return it("should pass options to PersistorManager", function(done) { - return this.handler.getFile(this.bucket, this.key, {start: 0, end: 8}, () => { - expect(this.PersistorManager.getFileStream.lastCall.args[2].start).to.equal(0); - expect(this.PersistorManager.getFileStream.lastCall.args[2].end).to.equal(8); - return done(); - }); - }); - }); + describe('_getConvertedFileAndCache', () => + it('should _convertFile ', function(done) { + this.stubbedStream = { something: 'here' } + this.localStream = { + on() {} + } + this.PersistorManager.sendFile = sinon.stub().callsArgWith(3) + this.LocalFileWriter.getStream = sinon + .stub() + .callsArgWith(1, null, this.localStream) + this.convetedKey = this.key + 'converted' + this.handler._convertFile = sinon + .stub() + .callsArgWith(3, null, this.stubbedPath) + this.ImageOptimiser.compressPng = sinon.stub().callsArgWith(1) + return this.handler._getConvertedFileAndCache( + this.bucket, + this.key, + this.convetedKey, + {}, + (err, fsStream) => { + this.handler._convertFile.called.should.equal(true) + this.PersistorManager.sendFile + .calledWith(this.bucket, this.convetedKey, this.stubbedPath) + .should.equal(true) + this.ImageOptimiser.compressPng + .calledWith(this.stubbedPath) + .should.equal(true) + this.LocalFileWriter.getStream + .calledWith(this.stubbedPath) + .should.equal(true) + fsStream.should.equal(this.localStream) + return done() + } + ) + })) + describe('_convertFile', function() { + beforeEach(function() { + this.FileConverter.convert.callsArgWith( + 2, + null, + this.formattedStubbedPath + ) + this.FileConverter.thumbnail.callsArgWith( + 1, + null, + this.formattedStubbedPath + ) + this.FileConverter.preview.callsArgWith( + 1, + null, + this.formattedStubbedPath + ) + this.handler._writeS3FileToDisk = sinon + .stub() + .callsArgWith(3, null, this.stubbedPath) + return this.LocalFileWriter.deleteFile.callsArgWith(1) + }) - describe("_getConvertedFile", function() { + it('should call thumbnail on the writer path if style was thumbnail was specified', function(done) { + return this.handler._convertFile( + this.bucket, + this.key, + { style: 'thumbnail' }, + (err, path) => { + path.should.equal(this.formattedStubbedPath) + this.FileConverter.thumbnail + .calledWith(this.stubbedPath) + .should.equal(true) + this.LocalFileWriter.deleteFile + .calledWith(this.stubbedPath) + .should.equal(true) + return done() + } + ) + }) - it("should getFileStream if it does exists", function(done){ - this.PersistorManager.checkIfFileExists.callsArgWith(2, null, true); - this.PersistorManager.getFileStream.callsArgWith(3); - return this.handler._getConvertedFile(this.bucket, this.key, {}, () => { - this.PersistorManager.getFileStream.calledWith(this.bucket).should.equal(true); - return done(); - }); - }); + it('should call preview on the writer path if style was preview was specified', function(done) { + return this.handler._convertFile( + this.bucket, + this.key, + { style: 'preview' }, + (err, path) => { + path.should.equal(this.formattedStubbedPath) + this.FileConverter.preview + .calledWith(this.stubbedPath) + .should.equal(true) + this.LocalFileWriter.deleteFile + .calledWith(this.stubbedPath) + .should.equal(true) + return done() + } + ) + }) - return it("should call _getConvertedFileAndCache if it does exists", function(done){ - this.PersistorManager.checkIfFileExists.callsArgWith(2, null, false); - this.handler._getConvertedFileAndCache = sinon.stub().callsArgWith(4); - return this.handler._getConvertedFile(this.bucket, this.key, {}, () => { - this.handler._getConvertedFileAndCache.calledWith(this.bucket, this.key).should.equal(true); - return done(); - }); - }); - }); + return it('should call convert on the writer path if a format was specified', function(done) { + return this.handler._convertFile( + this.bucket, + this.key, + { format: this.format }, + (err, path) => { + path.should.equal(this.formattedStubbedPath) + this.FileConverter.convert + .calledWith(this.stubbedPath, this.format) + .should.equal(true) + this.LocalFileWriter.deleteFile + .calledWith(this.stubbedPath) + .should.equal(true) + return done() + } + ) + }) + }) - describe("_getConvertedFileAndCache", () => it("should _convertFile ", function(done){ - this.stubbedStream = {"something":"here"}; - this.localStream = { - on() {} - }; - this.PersistorManager.sendFile = sinon.stub().callsArgWith(3); - this.LocalFileWriter.getStream = sinon.stub().callsArgWith(1, null, this.localStream); - this.convetedKey = this.key+"converted"; - this.handler._convertFile = sinon.stub().callsArgWith(3, null, this.stubbedPath); - this.ImageOptimiser.compressPng = sinon.stub().callsArgWith(1); - return this.handler._getConvertedFileAndCache(this.bucket, this.key, this.convetedKey, {}, (err, fsStream)=> { - this.handler._convertFile.called.should.equal(true); - this.PersistorManager.sendFile.calledWith(this.bucket, this.convetedKey, this.stubbedPath).should.equal(true); - this.ImageOptimiser.compressPng.calledWith(this.stubbedPath).should.equal(true); - this.LocalFileWriter.getStream.calledWith(this.stubbedPath).should.equal(true); - fsStream.should.equal(this.localStream); - return done(); - }); - })); + return describe('getDirectorySize', function() { + beforeEach(function() { + return this.PersistorManager.directorySize.callsArgWith(2) + }) - describe("_convertFile", function() { - beforeEach(function() { - this.FileConverter.convert.callsArgWith(2, null, this.formattedStubbedPath); - this.FileConverter.thumbnail.callsArgWith(1, null, this.formattedStubbedPath); - this.FileConverter.preview.callsArgWith(1, null, this.formattedStubbedPath); - this.handler._writeS3FileToDisk = sinon.stub().callsArgWith(3, null, this.stubbedPath); - return this.LocalFileWriter.deleteFile.callsArgWith(1); - }); - - it("should call thumbnail on the writer path if style was thumbnail was specified", function(done){ - return this.handler._convertFile(this.bucket, this.key, {style:"thumbnail"}, (err, path)=> { - path.should.equal(this.formattedStubbedPath); - this.FileConverter.thumbnail.calledWith(this.stubbedPath).should.equal(true); - this.LocalFileWriter.deleteFile.calledWith(this.stubbedPath).should.equal(true); - return done(); - }); - }); - - it("should call preview on the writer path if style was preview was specified", function(done){ - return this.handler._convertFile(this.bucket, this.key, {style:"preview"}, (err, path)=> { - path.should.equal(this.formattedStubbedPath); - this.FileConverter.preview.calledWith(this.stubbedPath).should.equal(true); - this.LocalFileWriter.deleteFile.calledWith(this.stubbedPath).should.equal(true); - return done(); - }); - }); - - return it("should call convert on the writer path if a format was specified", function(done){ - return this.handler._convertFile(this.bucket, this.key, {format:this.format}, (err, path)=> { - path.should.equal(this.formattedStubbedPath); - this.FileConverter.convert.calledWith(this.stubbedPath, this.format).should.equal(true); - this.LocalFileWriter.deleteFile.calledWith(this.stubbedPath).should.equal(true); - return done(); - }); - }); - }); - - return describe("getDirectorySize", function() { - - beforeEach(function() { - return this.PersistorManager.directorySize.callsArgWith(2); - }); - - return it("should call the filestore manager to get directory size", function(done){ - return this.handler.getDirectorySize(this.bucket, this.key, () => { - this.PersistorManager.directorySize.calledWith(this.bucket, this.key).should.equal(true); - return done(); - }); - }); - }); -}); + return it('should call the filestore manager to get directory size', function(done) { + return this.handler.getDirectorySize(this.bucket, this.key, () => { + this.PersistorManager.directorySize + .calledWith(this.bucket, this.key) + .should.equal(true) + return done() + }) + }) + }) +}) diff --git a/services/filestore/test/unit/js/ImageOptimiserTests.js b/services/filestore/test/unit/js/ImageOptimiserTests.js index 4e6cb858f4..6fdb09f89d 100644 --- a/services/filestore/test/unit/js/ImageOptimiserTests.js +++ b/services/filestore/test/unit/js/ImageOptimiserTests.js @@ -10,82 +10,72 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const { - assert -} = require("chai"); -const sinon = require('sinon'); -const chai = require('chai'); -const should = chai.should(); -const { - expect -} = chai; -const modulePath = "../../../app/js/ImageOptimiser.js"; -const SandboxedModule = require('sandboxed-module'); +const { assert } = require('chai') +const sinon = require('sinon') +const chai = require('chai') +const should = chai.should() +const { expect } = chai +const modulePath = '../../../app/js/ImageOptimiser.js' +const SandboxedModule = require('sandboxed-module') -describe("ImageOptimiser", function() { +describe('ImageOptimiser', function() { + beforeEach(function() { + this.child_process = { exec: sinon.stub() } + this.settings = { enableConversions: true } + this.optimiser = SandboxedModule.require(modulePath, { + requires: { + child_process: this.child_process, + 'logger-sharelatex': { + log() {}, + err() {}, + warn() {} + }, + 'settings-sharelatex': this.settings + } + }) - beforeEach(function() { - this.child_process = - {exec : sinon.stub()}; - this.settings = - {enableConversions:true}; - this.optimiser = SandboxedModule.require(modulePath, { requires: { - 'child_process': this.child_process, - "logger-sharelatex": { - log() {}, - err() {}, - warn() {} - }, - "settings-sharelatex": this.settings - } - } - ); - + this.sourcePath = '/this/path/here.eps' + return (this.error = 'Error') + }) - this.sourcePath = "/this/path/here.eps"; - return this.error = "Error"; - }); + describe('compressPng', function() { + it('convert the file', function(done) { + this.child_process.exec.callsArgWith(2) + return this.optimiser.compressPng(this.sourcePath, err => { + const args = this.child_process.exec.args[0][0] + args.should.equal(`optipng ${this.sourcePath}`) + return done() + }) + }) - describe("compressPng", function() { - + return it('should return the error', function(done) { + this.child_process.exec.callsArgWith(2, this.error) + return this.optimiser.compressPng(this.sourcePath, err => { + err.should.equal(this.error) + return done() + }) + }) + }) - it("convert the file", function(done){ - this.child_process.exec.callsArgWith(2); - return this.optimiser.compressPng(this.sourcePath, err=> { - const args = this.child_process.exec.args[0][0]; - args.should.equal(`optipng ${this.sourcePath}`); - return done(); - }); - }); + describe('when enableConversions is disabled', () => + it('should produce an error', function(done) { + this.settings.enableConversions = false + this.child_process.exec.callsArgWith(2) + return this.optimiser.compressPng(this.sourcePath, err => { + this.child_process.exec.called.should.equal(false) + expect(err).to.exist + return done() + }) + })) - - return it("should return the error", function(done){ - this.child_process.exec.callsArgWith(2, this.error); - return this.optimiser.compressPng(this.sourcePath, err=> { - err.should.equal(this.error); - return done(); - }); - }); - }); - - describe('when enableConversions is disabled', () => it('should produce an error', function(done) { - this.settings.enableConversions = false; - this.child_process.exec.callsArgWith(2); - return this.optimiser.compressPng(this.sourcePath, err=> { - this.child_process.exec.called.should.equal(false); - expect(err).to.exist; - return done(); - }); - })); - - - return describe('when optimiser is sigkilled', () => it('should not produce an error', function(done) { - this.error = new Error('woops'); - this.error.signal = 'SIGKILL'; - this.child_process.exec.callsArgWith(2, this.error); - return this.optimiser.compressPng(this.sourcePath, err=> { - expect(err).to.equal(null); - return done(); - }); - })); -}); + return describe('when optimiser is sigkilled', () => + it('should not produce an error', function(done) { + this.error = new Error('woops') + this.error.signal = 'SIGKILL' + this.child_process.exec.callsArgWith(2, this.error) + return this.optimiser.compressPng(this.sourcePath, err => { + expect(err).to.equal(null) + return done() + }) + })) +}) diff --git a/services/filestore/test/unit/js/KeybuilderTests.js b/services/filestore/test/unit/js/KeybuilderTests.js index 1e99899be7..09a0ea8717 100644 --- a/services/filestore/test/unit/js/KeybuilderTests.js +++ b/services/filestore/test/unit/js/KeybuilderTests.js @@ -10,56 +10,49 @@ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const { - assert -} = require("chai"); -const sinon = require('sinon'); -const chai = require('chai'); -const should = chai.should(); -const { - expect -} = chai; -const modulePath = "../../../app/js/KeyBuilder.js"; -const SandboxedModule = require('sandboxed-module'); +const { assert } = require('chai') +const sinon = require('sinon') +const chai = require('chai') +const should = chai.should() +const { expect } = chai +const modulePath = '../../../app/js/KeyBuilder.js' +const SandboxedModule = require('sandboxed-module') -describe("LocalFileWriter", function() { +describe('LocalFileWriter', function() { + beforeEach(function() { + this.keyBuilder = SandboxedModule.require(modulePath, { + requires: { + 'logger-sharelatex': { + log() {}, + err() {} + } + } + }) + return (this.key = '123/456') + }) - beforeEach(function() { + return describe('cachedKey', function() { + it('should add the fomat on', function() { + const opts = { format: 'png' } + const newKey = this.keyBuilder.addCachingToKey(this.key, opts) + return newKey.should.equal(`${this.key}-converted-cache/format-png`) + }) - this.keyBuilder = SandboxedModule.require(modulePath, { requires: { - "logger-sharelatex": { - log() {}, - err() {} - } - } - } - ); - return this.key = "123/456"; - }); - - return describe("cachedKey", function() { + it('should add the style on', function() { + const opts = { style: 'thumbnail' } + const newKey = this.keyBuilder.addCachingToKey(this.key, opts) + return newKey.should.equal(`${this.key}-converted-cache/style-thumbnail`) + }) - it("should add the fomat on", function() { - const opts = - {format: "png"}; - const newKey = this.keyBuilder.addCachingToKey(this.key, opts); - return newKey.should.equal(`${this.key}-converted-cache/format-png`); - }); - - it("should add the style on", function() { - const opts = - {style: "thumbnail"}; - const newKey = this.keyBuilder.addCachingToKey(this.key, opts); - return newKey.should.equal(`${this.key}-converted-cache/style-thumbnail`); - }); - - return it("should add format on first", function() { - const opts = { - style: "thumbnail", - format: "png" - }; - const newKey = this.keyBuilder.addCachingToKey(this.key, opts); - return newKey.should.equal(`${this.key}-converted-cache/format-png-style-thumbnail`); - }); - }); -}); + return it('should add format on first', function() { + const opts = { + style: 'thumbnail', + format: 'png' + } + const newKey = this.keyBuilder.addCachingToKey(this.key, opts) + return newKey.should.equal( + `${this.key}-converted-cache/format-png-style-thumbnail` + ) + }) + }) +}) diff --git a/services/filestore/test/unit/js/LocalFileWriterTests.js b/services/filestore/test/unit/js/LocalFileWriterTests.js index 1bf131a3c4..e7ea993ab9 100644 --- a/services/filestore/test/unit/js/LocalFileWriterTests.js +++ b/services/filestore/test/unit/js/LocalFileWriterTests.js @@ -11,118 +11,112 @@ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const { - assert -} = require("chai"); -const sinon = require('sinon'); -const chai = require('chai'); -const should = chai.should(); -const { - expect -} = chai; -const modulePath = "../../../app/js/LocalFileWriter.js"; -const SandboxedModule = require('sandboxed-module'); +const { assert } = require('chai') +const sinon = require('sinon') +const chai = require('chai') +const should = chai.should() +const { expect } = chai +const modulePath = '../../../app/js/LocalFileWriter.js' +const SandboxedModule = require('sandboxed-module') -describe("LocalFileWriter", function() { +describe('LocalFileWriter', function() { + beforeEach(function() { + this.writeStream = { + on(type, cb) { + if (type === 'finish') { + return cb() + } + } + } + this.readStream = { on() {} } + this.fs = { + createWriteStream: sinon.stub().returns(this.writeStream), + createReadStream: sinon.stub().returns(this.readStream), + unlink: sinon.stub() + } + this.settings = { + path: { + uploadFolder: 'somewhere' + } + } + this.writer = SandboxedModule.require(modulePath, { + requires: { + fs: this.fs, + 'logger-sharelatex': { + log() {}, + err() {} + }, + 'settings-sharelatex': this.settings, + 'metrics-sharelatex': { + inc() {}, + Timer() { + return { done() {} } + } + } + } + }) - beforeEach(function() { + return (this.stubbedFsPath = 'something/uploads/eio2k1j3') + }) - this.writeStream = { - on(type, cb){ - if (type === "finish") { - return cb(); - } - } - }; - this.readStream = - {on() {}}; - this.fs = { - createWriteStream : sinon.stub().returns(this.writeStream), - createReadStream: sinon.stub().returns(this.readStream), - unlink: sinon.stub() - }; - this.settings = { - path: { - uploadFolder:"somewhere" - } - }; - this.writer = SandboxedModule.require(modulePath, { requires: { - "fs": this.fs, - "logger-sharelatex": { - log() {}, - err() {} - }, - "settings-sharelatex":this.settings, - "metrics-sharelatex": { - inc() {}, - Timer() { - return {done() {}}; - } - } - } - } - ); + describe('writeStrem', function() { + beforeEach(function() { + return (this.writer._getPath = sinon.stub().returns(this.stubbedFsPath)) + }) - return this.stubbedFsPath = "something/uploads/eio2k1j3"; - }); + it('write the stream to ./uploads', function(done) { + const stream = { + pipe: dest => { + dest.should.equal(this.writeStream) + return done() + }, + on() {} + } + return this.writer.writeStream(stream, null, () => {}) + }) - describe("writeStrem", function() { - beforeEach(function() { - return this.writer._getPath = sinon.stub().returns(this.stubbedFsPath); - }); + return it('should send the path in the callback', function(done) { + const stream = { + pipe: dest => {}, + on(type, cb) { + if (type === 'end') { + return cb() + } + } + } + return this.writer.writeStream(stream, null, (err, fsPath) => { + fsPath.should.equal(this.stubbedFsPath) + return done() + }) + }) + }) - it("write the stream to ./uploads", function(done){ - const stream = { - pipe: dest=> { - dest.should.equal(this.writeStream); - return done(); - }, - on() {} - }; - return this.writer.writeStream(stream, null, ()=> {}); - }); + describe('getStream', function() { + it('should read the stream from the file ', function(done) { + return this.writer.getStream(this.stubbedFsPath, (err, stream) => { + this.fs.createReadStream + .calledWith(this.stubbedFsPath) + .should.equal(true) + return done() + }) + }) - return it("should send the path in the callback", function(done){ - const stream = { - pipe: dest=> {}, - on(type, cb){ - if (type === "end") { - return cb(); - } - } - }; - return this.writer.writeStream(stream, null, (err, fsPath)=> { - fsPath.should.equal(this.stubbedFsPath); - return done(); - }); - }); - }); - - describe("getStream", function() { - - it("should read the stream from the file ", function(done){ - return this.writer.getStream(this.stubbedFsPath, (err, stream)=> { - this.fs.createReadStream.calledWith(this.stubbedFsPath).should.equal(true); - return done(); - }); - }); - - return it("should send the stream in the callback", function(done){ - return this.writer.getStream(this.stubbedFsPath, (err, readStream)=> { - readStream.should.equal(this.readStream); - return done(); - }); - }); - }); - - return describe("delete file", () => it("should unlink the file", function(done){ - const error = "my error"; - this.fs.unlink.callsArgWith(1, error); - return this.writer.deleteFile(this.stubbedFsPath, err=> { - this.fs.unlink.calledWith(this.stubbedFsPath).should.equal(true); - err.should.equal(error); - return done(); - }); - })); -}); + return it('should send the stream in the callback', function(done) { + return this.writer.getStream(this.stubbedFsPath, (err, readStream) => { + readStream.should.equal(this.readStream) + return done() + }) + }) + }) + return describe('delete file', () => + it('should unlink the file', function(done) { + const error = 'my error' + this.fs.unlink.callsArgWith(1, error) + return this.writer.deleteFile(this.stubbedFsPath, err => { + this.fs.unlink.calledWith(this.stubbedFsPath).should.equal(true) + err.should.equal(error) + return done() + }) + })) +}) diff --git a/services/filestore/test/unit/js/PersistorManagerTests.js b/services/filestore/test/unit/js/PersistorManagerTests.js index 620d6f0083..ff49c05ce9 100644 --- a/services/filestore/test/unit/js/PersistorManagerTests.js +++ b/services/filestore/test/unit/js/PersistorManagerTests.js @@ -9,129 +9,129 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const logger = require("logger-sharelatex"); -const { - assert -} = require("chai"); -const sinon = require('sinon'); -const chai = require('chai'); -const should = chai.should(); -const { - expect -} = chai; -const modulePath = "../../../app/js/PersistorManager.js"; -const SandboxedModule = require('sandboxed-module'); +const logger = require('logger-sharelatex') +const { assert } = require('chai') +const sinon = require('sinon') +const chai = require('chai') +const should = chai.should() +const { expect } = chai +const modulePath = '../../../app/js/PersistorManager.js' +const SandboxedModule = require('sandboxed-module') +describe('PersistorManagerTests', function() { + beforeEach(function() { + return (this.S3PersistorManager = { + getFileStream: sinon.stub(), + checkIfFileExists: sinon.stub(), + deleteFile: sinon.stub(), + deleteDirectory: sinon.stub(), + sendStream: sinon.stub(), + insertFile: sinon.stub() + }) + }) -describe("PersistorManagerTests", function() { - - beforeEach(function() { - return this.S3PersistorManager = { - getFileStream: sinon.stub(), - checkIfFileExists: sinon.stub(), - deleteFile: sinon.stub(), - deleteDirectory: sinon.stub(), - sendStream: sinon.stub(), - insertFile: sinon.stub() - }; - }); - - describe("test s3 mixin", function() { - beforeEach(function() { - this.settings = { - filestore: { - backend: "s3" - } - }; - this.requires = { - "./S3PersistorManager": this.S3PersistorManager, - "settings-sharelatex": this.settings, - "logger-sharelatex": { - log() {}, - err() {} - } - }; - return this.PersistorManager = SandboxedModule.require(modulePath, {requires: this.requires}); - }); - - it("should load getFileStream", function(done) { - this.PersistorManager.should.respondTo("getFileStream"); - this.PersistorManager.getFileStream(); - this.S3PersistorManager.getFileStream.calledOnce.should.equal(true); - return done(); - }); - - it("should load checkIfFileExists", function(done) { - this.PersistorManager.checkIfFileExists(); - this.S3PersistorManager.checkIfFileExists.calledOnce.should.equal(true); - return done(); - }); - - it("should load deleteFile", function(done) { - this.PersistorManager.deleteFile(); - this.S3PersistorManager.deleteFile.calledOnce.should.equal(true); - return done(); - }); - - it("should load deleteDirectory", function(done) { - this.PersistorManager.deleteDirectory(); - this.S3PersistorManager.deleteDirectory.calledOnce.should.equal(true); - return done(); - }); - - it("should load sendStream", function(done) { - this.PersistorManager.sendStream(); - this.S3PersistorManager.sendStream.calledOnce.should.equal(true); - return done(); - }); - - return it("should load insertFile", function(done) { - this.PersistorManager.insertFile(); - this.S3PersistorManager.insertFile.calledOnce.should.equal(true); - return done(); - }); - }); - - describe("test unspecified mixins", () => it("should load s3 when no wrapper specified", function(done) { - this.settings = {filestore:{}}; - this.requires = { - "./S3PersistorManager": this.S3PersistorManager, - "settings-sharelatex": this.settings, - "logger-sharelatex": { - log() {}, - err() {} - } - }; - this.PersistorManager = SandboxedModule.require(modulePath, {requires: this.requires}); - this.PersistorManager.should.respondTo("getFileStream"); - this.PersistorManager.getFileStream(); - this.S3PersistorManager.getFileStream.calledOnce.should.equal(true); - return done(); - })); - - return describe("test invalid mixins", () => it("should not load an invalid wrapper", function(done) { - this.settings = { - filestore: { - backend:"magic" - } - }; - this.requires = { - "./S3PersistorManager": this.S3PersistorManager, - "settings-sharelatex": this.settings, - "logger-sharelatex": { - log() {}, - err() {} - } - }; - this.fsWrapper=null; - try { - this.PersistorManager=SandboxedModule.require(modulePath, {requires: this.requires}); - } catch (error) { - assert.equal("Unknown filestore backend: magic",error.message); + describe('test s3 mixin', function() { + beforeEach(function() { + this.settings = { + filestore: { + backend: 's3' } - assert.isNull(this.fsWrapper); - return done(); - })); -}); + } + this.requires = { + './S3PersistorManager': this.S3PersistorManager, + 'settings-sharelatex': this.settings, + 'logger-sharelatex': { + log() {}, + err() {} + } + } + return (this.PersistorManager = SandboxedModule.require(modulePath, { + requires: this.requires + })) + }) + it('should load getFileStream', function(done) { + this.PersistorManager.should.respondTo('getFileStream') + this.PersistorManager.getFileStream() + this.S3PersistorManager.getFileStream.calledOnce.should.equal(true) + return done() + }) + it('should load checkIfFileExists', function(done) { + this.PersistorManager.checkIfFileExists() + this.S3PersistorManager.checkIfFileExists.calledOnce.should.equal(true) + return done() + }) + + it('should load deleteFile', function(done) { + this.PersistorManager.deleteFile() + this.S3PersistorManager.deleteFile.calledOnce.should.equal(true) + return done() + }) + + it('should load deleteDirectory', function(done) { + this.PersistorManager.deleteDirectory() + this.S3PersistorManager.deleteDirectory.calledOnce.should.equal(true) + return done() + }) + + it('should load sendStream', function(done) { + this.PersistorManager.sendStream() + this.S3PersistorManager.sendStream.calledOnce.should.equal(true) + return done() + }) + + return it('should load insertFile', function(done) { + this.PersistorManager.insertFile() + this.S3PersistorManager.insertFile.calledOnce.should.equal(true) + return done() + }) + }) + + describe('test unspecified mixins', () => + it('should load s3 when no wrapper specified', function(done) { + this.settings = { filestore: {} } + this.requires = { + './S3PersistorManager': this.S3PersistorManager, + 'settings-sharelatex': this.settings, + 'logger-sharelatex': { + log() {}, + err() {} + } + } + this.PersistorManager = SandboxedModule.require(modulePath, { + requires: this.requires + }) + this.PersistorManager.should.respondTo('getFileStream') + this.PersistorManager.getFileStream() + this.S3PersistorManager.getFileStream.calledOnce.should.equal(true) + return done() + })) + + return describe('test invalid mixins', () => + it('should not load an invalid wrapper', function(done) { + this.settings = { + filestore: { + backend: 'magic' + } + } + this.requires = { + './S3PersistorManager': this.S3PersistorManager, + 'settings-sharelatex': this.settings, + 'logger-sharelatex': { + log() {}, + err() {} + } + } + this.fsWrapper = null + try { + this.PersistorManager = SandboxedModule.require(modulePath, { + requires: this.requires + }) + } catch (error) { + assert.equal('Unknown filestore backend: magic', error.message) + } + assert.isNull(this.fsWrapper) + return done() + })) +}) diff --git a/services/filestore/test/unit/js/S3PersistorManagerTests.js b/services/filestore/test/unit/js/S3PersistorManagerTests.js index a3a7d16825..c0a03696a8 100644 --- a/services/filestore/test/unit/js/S3PersistorManagerTests.js +++ b/services/filestore/test/unit/js/S3PersistorManagerTests.js @@ -11,444 +11,609 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const { - assert -} = require("chai"); -const sinon = require('sinon'); -const chai = require('chai'); -const should = chai.should(); -const { - expect -} = chai; -const modulePath = "../../../app/js/S3PersistorManager.js"; -const SandboxedModule = require('sandboxed-module'); +const { assert } = require('chai') +const sinon = require('sinon') +const chai = require('chai') +const should = chai.should() +const { expect } = chai +const modulePath = '../../../app/js/S3PersistorManager.js' +const SandboxedModule = require('sandboxed-module') -describe("S3PersistorManagerTests", function() { +describe('S3PersistorManagerTests', function() { + beforeEach(function() { + this.settings = { + filestore: { + backend: 's3', + s3: { + secret: 'secret', + key: 'this_key' + }, + stores: { + user_files: 'sl_user_files' + } + } + } + this.knoxClient = { + putFile: sinon.stub(), + copyFile: sinon.stub(), + list: sinon.stub(), + deleteMultiple: sinon.stub(), + get: sinon.stub() + } + this.knox = { createClient: sinon.stub().returns(this.knoxClient) } + this.s3EventHandlers = {} + this.s3Request = { + on: sinon.stub().callsFake((event, callback) => { + return (this.s3EventHandlers[event] = callback) + }), + send: sinon.stub() + } + this.s3Response = { + httpResponse: { + createUnbufferedStream: sinon.stub() + } + } + this.s3Client = { + copyObject: sinon.stub(), + headObject: sinon.stub(), + getObject: sinon.stub().returns(this.s3Request) + } + this.awsS3 = sinon.stub().returns(this.s3Client) + this.LocalFileWriter = { + writeStream: sinon.stub(), + deleteFile: sinon.stub() + } + this.request = sinon.stub() + this.requires = { + knox: this.knox, + 'aws-sdk/clients/s3': this.awsS3, + 'settings-sharelatex': this.settings, + './LocalFileWriter': this.LocalFileWriter, + 'logger-sharelatex': { + log() {}, + err() {} + }, + request: this.request, + './Errors': (this.Errors = { NotFoundError: sinon.stub() }) + } + this.key = 'my/key' + this.bucketName = 'my-bucket' + this.error = 'my errror' + return (this.S3PersistorManager = SandboxedModule.require(modulePath, { + requires: this.requires + })) + }) - beforeEach(function() { - this.settings = { - filestore: { - backend: "s3", - s3: { - secret: "secret", - key: "this_key" - }, - stores: { - user_files:"sl_user_files" - } - } - }; - this.knoxClient = { - putFile:sinon.stub(), - copyFile:sinon.stub(), - list: sinon.stub(), - deleteMultiple: sinon.stub(), - get: sinon.stub() - }; - this.knox = - {createClient: sinon.stub().returns(this.knoxClient)}; - this.s3EventHandlers = {}; - this.s3Request = { - on: sinon.stub().callsFake((event, callback) => { - return this.s3EventHandlers[event] = callback; - }), - send: sinon.stub() - }; - this.s3Response = { - httpResponse: { - createUnbufferedStream: sinon.stub() - } - }; - this.s3Client = { - copyObject: sinon.stub(), - headObject: sinon.stub(), - getObject: sinon.stub().returns(this.s3Request) - }; - this.awsS3 = sinon.stub().returns(this.s3Client); - this.LocalFileWriter = { - writeStream: sinon.stub(), - deleteFile: sinon.stub() - }; - this.request = sinon.stub(); - this.requires = { - "knox": this.knox, - "aws-sdk/clients/s3": this.awsS3, - "settings-sharelatex": this.settings, - "./LocalFileWriter":this.LocalFileWriter, - "logger-sharelatex": { - log() {}, - err() {} - }, - "request": this.request, - "./Errors": (this.Errors = - {NotFoundError: sinon.stub()}) - }; - this.key = "my/key"; - this.bucketName = "my-bucket"; - this.error = "my errror"; - return this.S3PersistorManager = SandboxedModule.require(modulePath, {requires: this.requires}); - }); + describe('getFileStream', function() { + describe('success', function() { + beforeEach(function() { + this.expectedStream = { expectedStream: true } + this.expectedStream.on = sinon.stub() + this.s3Request.send.callsFake(() => { + return this.s3EventHandlers.httpHeaders( + 200, + {}, + this.s3Response, + 'OK' + ) + }) + return this.s3Response.httpResponse.createUnbufferedStream.returns( + this.expectedStream + ) + }) - describe("getFileStream", function() { - describe("success", function() { - beforeEach(function() { - this.expectedStream = { expectedStream: true }; - this.expectedStream.on = sinon.stub(); - this.s3Request.send.callsFake(() => { - return this.s3EventHandlers.httpHeaders(200, {}, this.s3Response, "OK"); - }); - return this.s3Response.httpResponse.createUnbufferedStream.returns(this.expectedStream); - }); + it('returns a stream', function(done) { + return this.S3PersistorManager.getFileStream( + this.bucketName, + this.key, + {}, + (err, stream) => { + if (err != null) { + return done(err) + } + expect(stream).to.equal(this.expectedStream) + return done() + } + ) + }) - it("returns a stream", function(done) { - return this.S3PersistorManager.getFileStream(this.bucketName, this.key, {}, (err, stream) => { - if (err != null) { - return done(err); - } - expect(stream).to.equal(this.expectedStream); - return done(); - }); - }); + it('sets the AWS client up with credentials from settings', function(done) { + return this.S3PersistorManager.getFileStream( + this.bucketName, + this.key, + {}, + (err, stream) => { + if (err != null) { + return done(err) + } + expect(this.awsS3.lastCall.args).to.deep.equal([ + { + credentials: { + accessKeyId: this.settings.filestore.s3.key, + secretAccessKey: this.settings.filestore.s3.secret + } + } + ]) + return done() + } + ) + }) - it("sets the AWS client up with credentials from settings", function(done) { - return this.S3PersistorManager.getFileStream(this.bucketName, this.key, {}, (err, stream) => { - if (err != null) { - return done(err); - } - expect(this.awsS3.lastCall.args).to.deep.equal([{ - credentials: { - accessKeyId: this.settings.filestore.s3.key, - secretAccessKey: this.settings.filestore.s3.secret - } - }]); - return done(); - }); - }); + it('fetches the right key from the right bucket', function(done) { + return this.S3PersistorManager.getFileStream( + this.bucketName, + this.key, + {}, + (err, stream) => { + if (err != null) { + return done(err) + } + expect(this.s3Client.getObject.lastCall.args).to.deep.equal([ + { + Bucket: this.bucketName, + Key: this.key + } + ]) + return done() + } + ) + }) - it("fetches the right key from the right bucket", function(done) { - return this.S3PersistorManager.getFileStream(this.bucketName, this.key, {}, (err, stream) => { - if (err != null) { - return done(err); - } - expect(this.s3Client.getObject.lastCall.args).to.deep.equal([{ - Bucket: this.bucketName, - Key: this.key - }]); - return done(); - }); - }); + it('accepts alternative credentials', function(done) { + const accessKeyId = 'that_key' + const secret = 'that_secret' + const opts = { + credentials: { + auth_key: accessKeyId, + auth_secret: secret + } + } + return this.S3PersistorManager.getFileStream( + this.bucketName, + this.key, + opts, + (err, stream) => { + if (err != null) { + return done(err) + } + expect(this.awsS3.lastCall.args).to.deep.equal([ + { + credentials: { + accessKeyId, + secretAccessKey: secret + } + } + ]) + expect(stream).to.equal(this.expectedStream) + return done() + } + ) + }) - it("accepts alternative credentials", function(done) { - const accessKeyId = "that_key"; - const secret = "that_secret"; - const opts = { - credentials: { - auth_key: accessKeyId, - auth_secret: secret - } - }; - return this.S3PersistorManager.getFileStream(this.bucketName, this.key, opts, (err, stream) => { - if (err != null) { - return done(err); - } - expect(this.awsS3.lastCall.args).to.deep.equal([{ - credentials: { - accessKeyId, - secretAccessKey: secret - } - }]); - expect(stream).to.equal(this.expectedStream); - return done(); - }); - }); + return it('accepts byte range', function(done) { + const start = 0 + const end = 8 + const opts = { start, end } + return this.S3PersistorManager.getFileStream( + this.bucketName, + this.key, + opts, + (err, stream) => { + if (err != null) { + return done(err) + } + expect(this.s3Client.getObject.lastCall.args).to.deep.equal([ + { + Bucket: this.bucketName, + Key: this.key, + Range: `bytes=${start}-${end}` + } + ]) + expect(stream).to.equal(this.expectedStream) + return done() + } + ) + }) + }) - return it("accepts byte range", function(done) { - const start = 0; - const end = 8; - const opts = { start, end }; - return this.S3PersistorManager.getFileStream(this.bucketName, this.key, opts, (err, stream) => { - if (err != null) { - return done(err); - } - expect(this.s3Client.getObject.lastCall.args).to.deep.equal([{ - Bucket: this.bucketName, - Key: this.key, - Range: `bytes=${start}-${end}` - }]); - expect(stream).to.equal(this.expectedStream); - return done(); - }); - }); - }); + return describe('errors', function() { + describe("when the file doesn't exist", function() { + beforeEach(function() { + return this.s3Request.send.callsFake(() => { + return this.s3EventHandlers.httpHeaders( + 404, + {}, + this.s3Response, + 'Not found' + ) + }) + }) - return describe("errors", function() { - describe("when the file doesn't exist", function() { - beforeEach(function() { - return this.s3Request.send.callsFake(() => { - return this.s3EventHandlers.httpHeaders(404, {}, this.s3Response, "Not found"); - }); - }); + return it('returns a NotFoundError that indicates the bucket and key', function(done) { + return this.S3PersistorManager.getFileStream( + this.bucketName, + this.key, + {}, + (err, stream) => { + expect(err).to.be.instanceof(this.Errors.NotFoundError) + const errMsg = this.Errors.NotFoundError.lastCall.args[0] + expect(errMsg).to.match(new RegExp(`.*${this.bucketName}.*`)) + expect(errMsg).to.match(new RegExp(`.*${this.key}.*`)) + return done() + } + ) + }) + }) - return it("returns a NotFoundError that indicates the bucket and key", function(done) { - return this.S3PersistorManager.getFileStream(this.bucketName, this.key, {}, (err, stream) => { - expect(err).to.be.instanceof(this.Errors.NotFoundError); - const errMsg = this.Errors.NotFoundError.lastCall.args[0]; - expect(errMsg).to.match(new RegExp(`.*${this.bucketName}.*`)); - expect(errMsg).to.match(new RegExp(`.*${this.key}.*`)); - return done(); - }); - }); - }); + describe('when S3 encounters an unkown error', function() { + beforeEach(function() { + return this.s3Request.send.callsFake(() => { + return this.s3EventHandlers.httpHeaders( + 500, + {}, + this.s3Response, + 'Internal server error' + ) + }) + }) - describe("when S3 encounters an unkown error", function() { - beforeEach(function() { - return this.s3Request.send.callsFake(() => { - return this.s3EventHandlers.httpHeaders(500, {}, this.s3Response, "Internal server error"); - }); - }); + return it('returns an error', function(done) { + return this.S3PersistorManager.getFileStream( + this.bucketName, + this.key, + {}, + (err, stream) => { + expect(err).to.be.instanceof(Error) + return done() + } + ) + }) + }) - return it("returns an error", function(done) { - return this.S3PersistorManager.getFileStream(this.bucketName, this.key, {}, (err, stream) => { - expect(err).to.be.instanceof(Error); - return done(); - }); - }); - }); + return describe('when the S3 request errors out before receiving HTTP headers', function() { + beforeEach(function() { + return this.s3Request.send.callsFake(() => { + return this.s3EventHandlers.error(new Error('connection failed')) + }) + }) - return describe("when the S3 request errors out before receiving HTTP headers", function() { - beforeEach(function() { - return this.s3Request.send.callsFake(() => { - return this.s3EventHandlers.error(new Error("connection failed")); - }); - }); + return it('returns an error', function(done) { + return this.S3PersistorManager.getFileStream( + this.bucketName, + this.key, + {}, + (err, stream) => { + expect(err).to.be.instanceof(Error) + return done() + } + ) + }) + }) + }) + }) - return it("returns an error", function(done) { - return this.S3PersistorManager.getFileStream(this.bucketName, this.key, {}, (err, stream) => { - expect(err).to.be.instanceof(Error); - return done(); - }); - }); - }); - }); - }); + describe('getFileSize', function() { + it('should obtain the file size from S3', function(done) { + const expectedFileSize = 123 + this.s3Client.headObject.yields( + new Error('s3Client.headObject got unexpected arguments') + ) + this.s3Client.headObject + .withArgs({ + Bucket: this.bucketName, + Key: this.key + }) + .yields(null, { ContentLength: expectedFileSize }) - describe("getFileSize", function() { - it("should obtain the file size from S3", function(done) { - const expectedFileSize = 123; - this.s3Client.headObject.yields(new Error( - "s3Client.headObject got unexpected arguments" - )); - this.s3Client.headObject.withArgs({ - Bucket: this.bucketName, - Key: this.key - }).yields(null, { ContentLength: expectedFileSize }); + return this.S3PersistorManager.getFileSize( + this.bucketName, + this.key, + (err, fileSize) => { + if (err != null) { + return done(err) + } + expect(fileSize).to.equal(expectedFileSize) + return done() + } + ) + }) - return this.S3PersistorManager.getFileSize(this.bucketName, this.key, (err, fileSize) => { - if (err != null) { - return done(err); - } - expect(fileSize).to.equal(expectedFileSize); - return done(); - }); - }); + ;[403, 404].forEach(statusCode => + it(`should throw NotFoundError when S3 responds with ${statusCode}`, function(done) { + const error = new Error() + error.statusCode = statusCode + this.s3Client.headObject.yields(error) - [403, 404].forEach(statusCode => it(`should throw NotFoundError when S3 responds with ${statusCode}`, function(done) { - const error = new Error(); - error.statusCode = statusCode; - this.s3Client.headObject.yields(error); + return this.S3PersistorManager.getFileSize( + this.bucketName, + this.key, + (err, fileSize) => { + expect(err).to.be.an.instanceof(this.Errors.NotFoundError) + return done() + } + ) + }) + ) - return this.S3PersistorManager.getFileSize(this.bucketName, this.key, (err, fileSize) => { - expect(err).to.be.an.instanceof(this.Errors.NotFoundError); - return done(); - }); - })); + return it('should rethrow any other error', function(done) { + const error = new Error() + this.s3Client.headObject.yields(error) + this.s3Client.headObject.yields(error) - return it("should rethrow any other error", function(done) { - const error = new Error(); - this.s3Client.headObject.yields(error); - this.s3Client.headObject.yields(error); + return this.S3PersistorManager.getFileSize( + this.bucketName, + this.key, + (err, fileSize) => { + expect(err).to.equal(error) + return done() + } + ) + }) + }) - return this.S3PersistorManager.getFileSize(this.bucketName, this.key, (err, fileSize) => { - expect(err).to.equal(error); - return done(); - }); - }); - }); + describe('sendFile', function() { + beforeEach(function() { + return this.knoxClient.putFile.returns({ on() {} }) + }) - describe("sendFile", function() { + it('should put file with knox', function(done) { + this.LocalFileWriter.deleteFile.callsArgWith(1) + this.knoxClient.putFile.callsArgWith(2, this.error) + return this.S3PersistorManager.sendFile( + this.bucketName, + this.key, + this.fsPath, + err => { + this.knoxClient.putFile + .calledWith(this.fsPath, this.key) + .should.equal(true) + err.should.equal(this.error) + return done() + } + ) + }) - beforeEach(function() { - return this.knoxClient.putFile.returns({on() {}}); - }); + return it('should delete the file and pass the error with it', function(done) { + this.LocalFileWriter.deleteFile.callsArgWith(1) + this.knoxClient.putFile.callsArgWith(2, this.error) + return this.S3PersistorManager.sendFile( + this.bucketName, + this.key, + this.fsPath, + err => { + this.knoxClient.putFile + .calledWith(this.fsPath, this.key) + .should.equal(true) + err.should.equal(this.error) + return done() + } + ) + }) + }) - it("should put file with knox", function(done){ - this.LocalFileWriter.deleteFile.callsArgWith(1); - this.knoxClient.putFile.callsArgWith(2, this.error); - return this.S3PersistorManager.sendFile(this.bucketName, this.key, this.fsPath, err=> { - this.knoxClient.putFile.calledWith(this.fsPath, this.key).should.equal(true); - err.should.equal(this.error); - return done(); - }); - }); + describe('sendStream', function() { + beforeEach(function() { + this.fsPath = 'to/some/where' + this.origin = { on() {} } + return (this.S3PersistorManager.sendFile = sinon.stub().callsArgWith(3)) + }) - return it("should delete the file and pass the error with it", function(done){ - this.LocalFileWriter.deleteFile.callsArgWith(1); - this.knoxClient.putFile.callsArgWith(2, this.error); - return this.S3PersistorManager.sendFile(this.bucketName, this.key, this.fsPath, err=> { - this.knoxClient.putFile.calledWith(this.fsPath, this.key).should.equal(true); - err.should.equal(this.error); - return done(); - }); - }); - }); + it('should send stream to LocalFileWriter', function(done) { + this.LocalFileWriter.deleteFile.callsArgWith(1) + this.LocalFileWriter.writeStream.callsArgWith(2, null, this.fsPath) + return this.S3PersistorManager.sendStream( + this.bucketName, + this.key, + this.origin, + () => { + this.LocalFileWriter.writeStream + .calledWith(this.origin) + .should.equal(true) + return done() + } + ) + }) - describe("sendStream", function() { - beforeEach(function() { - this.fsPath = "to/some/where"; - this.origin = - {on() {}}; - return this.S3PersistorManager.sendFile = sinon.stub().callsArgWith(3); - }); + it('should return the error from LocalFileWriter', function(done) { + this.LocalFileWriter.deleteFile.callsArgWith(1) + this.LocalFileWriter.writeStream.callsArgWith(2, this.error) + return this.S3PersistorManager.sendStream( + this.bucketName, + this.key, + this.origin, + err => { + err.should.equal(this.error) + return done() + } + ) + }) - it("should send stream to LocalFileWriter", function(done){ - this.LocalFileWriter.deleteFile.callsArgWith(1); - this.LocalFileWriter.writeStream.callsArgWith(2, null, this.fsPath); - return this.S3PersistorManager.sendStream(this.bucketName, this.key, this.origin, () => { - this.LocalFileWriter.writeStream.calledWith(this.origin).should.equal(true); - return done(); - }); - }); + return it('should send the file to the filestore', function(done) { + this.LocalFileWriter.deleteFile.callsArgWith(1) + this.LocalFileWriter.writeStream.callsArgWith(2) + return this.S3PersistorManager.sendStream( + this.bucketName, + this.key, + this.origin, + err => { + this.S3PersistorManager.sendFile.called.should.equal(true) + return done() + } + ) + }) + }) - it("should return the error from LocalFileWriter", function(done){ - this.LocalFileWriter.deleteFile.callsArgWith(1); - this.LocalFileWriter.writeStream.callsArgWith(2, this.error); - return this.S3PersistorManager.sendStream(this.bucketName, this.key, this.origin, err=> { - err.should.equal(this.error); - return done(); - }); - }); + describe('copyFile', function() { + beforeEach(function() { + this.sourceKey = 'my/key' + return (this.destKey = 'my/dest/key') + }) - return it("should send the file to the filestore", function(done){ - this.LocalFileWriter.deleteFile.callsArgWith(1); - this.LocalFileWriter.writeStream.callsArgWith(2); - return this.S3PersistorManager.sendStream(this.bucketName, this.key, this.origin, err=> { - this.S3PersistorManager.sendFile.called.should.equal(true); - return done(); - }); - }); - }); + it('should use AWS SDK to copy file', function(done) { + this.s3Client.copyObject.callsArgWith(1, this.error) + return this.S3PersistorManager.copyFile( + this.bucketName, + this.sourceKey, + this.destKey, + err => { + err.should.equal(this.error) + this.s3Client.copyObject + .calledWith({ + Bucket: this.bucketName, + Key: this.destKey, + CopySource: this.bucketName + '/' + this.key + }) + .should.equal(true) + return done() + } + ) + }) - describe("copyFile", function() { - beforeEach(function() { - this.sourceKey = "my/key"; - return this.destKey = "my/dest/key"; - }); + return it('should return a NotFoundError object if the original file does not exist', function(done) { + const NoSuchKeyError = { code: 'NoSuchKey' } + this.s3Client.copyObject.callsArgWith(1, NoSuchKeyError) + return this.S3PersistorManager.copyFile( + this.bucketName, + this.sourceKey, + this.destKey, + err => { + expect(err instanceof this.Errors.NotFoundError).to.equal(true) + return done() + } + ) + }) + }) - it("should use AWS SDK to copy file", function(done){ - this.s3Client.copyObject.callsArgWith(1, this.error); - return this.S3PersistorManager.copyFile(this.bucketName, this.sourceKey, this.destKey, err=> { - err.should.equal(this.error); - this.s3Client.copyObject.calledWith({Bucket: this.bucketName, Key: this.destKey, CopySource: this.bucketName + '/' + this.key}).should.equal(true); - return done(); - }); - }); + describe('deleteDirectory', () => + it('should list the contents passing them onto multi delete', function(done) { + const data = { Contents: [{ Key: '1234' }, { Key: '456' }] } + this.knoxClient.list.callsArgWith(1, null, data) + this.knoxClient.deleteMultiple.callsArgWith(1) + return this.S3PersistorManager.deleteDirectory( + this.bucketName, + this.key, + err => { + this.knoxClient.deleteMultiple + .calledWith(['1234', '456']) + .should.equal(true) + return done() + } + ) + })) - return it("should return a NotFoundError object if the original file does not exist", function(done){ - const NoSuchKeyError = {code: "NoSuchKey"}; - this.s3Client.copyObject.callsArgWith(1, NoSuchKeyError); - return this.S3PersistorManager.copyFile(this.bucketName, this.sourceKey, this.destKey, err=> { - expect(err instanceof this.Errors.NotFoundError).to.equal(true); - return done(); - }); - }); - }); + describe('deleteFile', function() { + it('should use correct options', function(done) { + this.request.callsArgWith(1) - describe("deleteDirectory", () => it("should list the contents passing them onto multi delete", function(done){ - const data = - {Contents: [{Key:"1234"}, {Key: "456"}]}; - this.knoxClient.list.callsArgWith(1, null, data); - this.knoxClient.deleteMultiple.callsArgWith(1); - return this.S3PersistorManager.deleteDirectory(this.bucketName, this.key, err=> { - this.knoxClient.deleteMultiple.calledWith(["1234","456"]).should.equal(true); - return done(); - }); - })); + return this.S3PersistorManager.deleteFile( + this.bucketName, + this.key, + err => { + const opts = this.request.args[0][0] + assert.deepEqual(opts.aws, { + key: this.settings.filestore.s3.key, + secret: this.settings.filestore.s3.secret, + bucket: this.bucketName + }) + opts.method.should.equal('delete') + opts.timeout.should.equal(30 * 1000) + opts.uri.should.equal( + `https://${this.bucketName}.s3.amazonaws.com/${this.key}` + ) + return done() + } + ) + }) - describe("deleteFile", function() { + return it('should return the error', function(done) { + this.request.callsArgWith(1, this.error) - it("should use correct options", function(done){ - this.request.callsArgWith(1); + return this.S3PersistorManager.deleteFile( + this.bucketName, + this.key, + err => { + err.should.equal(this.error) + return done() + } + ) + }) + }) - return this.S3PersistorManager.deleteFile(this.bucketName, this.key, err=> { - const opts = this.request.args[0][0]; - assert.deepEqual(opts.aws, {key:this.settings.filestore.s3.key, secret:this.settings.filestore.s3.secret, bucket:this.bucketName}); - opts.method.should.equal("delete"); - opts.timeout.should.equal((30*1000)); - opts.uri.should.equal(`https://${this.bucketName}.s3.amazonaws.com/${this.key}`); - return done(); - }); - }); + describe('checkIfFileExists', function() { + it('should use correct options', function(done) { + this.request.callsArgWith(1, null, { statusCode: 200 }) - return it("should return the error", function(done){ - this.request.callsArgWith(1, this.error); + return this.S3PersistorManager.checkIfFileExists( + this.bucketName, + this.key, + err => { + const opts = this.request.args[0][0] + assert.deepEqual(opts.aws, { + key: this.settings.filestore.s3.key, + secret: this.settings.filestore.s3.secret, + bucket: this.bucketName + }) + opts.method.should.equal('head') + opts.timeout.should.equal(30 * 1000) + opts.uri.should.equal( + `https://${this.bucketName}.s3.amazonaws.com/${this.key}` + ) + return done() + } + ) + }) - return this.S3PersistorManager.deleteFile(this.bucketName, this.key, err=> { - err.should.equal(this.error); - return done(); - }); - }); - }); + it('should return true for a 200', function(done) { + this.request.callsArgWith(1, null, { statusCode: 200 }) - describe("checkIfFileExists", function() { + return this.S3PersistorManager.checkIfFileExists( + this.bucketName, + this.key, + (err, exists) => { + exists.should.equal(true) + return done() + } + ) + }) - it("should use correct options", function(done){ - this.request.callsArgWith(1, null, {statusCode:200}); + it('should return false for a non 200', function(done) { + this.request.callsArgWith(1, null, { statusCode: 404 }) - return this.S3PersistorManager.checkIfFileExists(this.bucketName, this.key, err=> { - const opts = this.request.args[0][0]; - assert.deepEqual(opts.aws, {key:this.settings.filestore.s3.key, secret:this.settings.filestore.s3.secret, bucket:this.bucketName}); - opts.method.should.equal("head"); - opts.timeout.should.equal((30*1000)); - opts.uri.should.equal(`https://${this.bucketName}.s3.amazonaws.com/${this.key}`); - return done(); - }); - }); + return this.S3PersistorManager.checkIfFileExists( + this.bucketName, + this.key, + (err, exists) => { + exists.should.equal(false) + return done() + } + ) + }) - it("should return true for a 200", function(done){ - this.request.callsArgWith(1, null, {statusCode:200}); + return it('should return the error', function(done) { + this.request.callsArgWith(1, this.error, {}) - return this.S3PersistorManager.checkIfFileExists(this.bucketName, this.key, (err, exists)=> { - exists.should.equal(true); - return done(); - }); - }); + return this.S3PersistorManager.checkIfFileExists( + this.bucketName, + this.key, + err => { + err.should.equal(this.error) + return done() + } + ) + }) + }) - it("should return false for a non 200", function(done){ - this.request.callsArgWith(1, null, {statusCode:404}); - - return this.S3PersistorManager.checkIfFileExists(this.bucketName, this.key, (err, exists)=> { - exists.should.equal(false); - return done(); - }); - }); - - return it("should return the error", function(done){ - this.request.callsArgWith(1, this.error, {}); - - return this.S3PersistorManager.checkIfFileExists(this.bucketName, this.key, err=> { - err.should.equal(this.error); - return done(); - }); - }); - }); - - return describe("directorySize", () => it("should sum directory files size", function(done) { - const data = - {Contents: [ {Size: 1024}, {Size: 2048} ]}; - this.knoxClient.list.callsArgWith(1, null, data); - return this.S3PersistorManager.directorySize(this.bucketName, this.key, (err, totalSize)=> { - totalSize.should.equal(3072); - return done(); - }); - })); -}); + return describe('directorySize', () => + it('should sum directory files size', function(done) { + const data = { Contents: [{ Size: 1024 }, { Size: 2048 }] } + this.knoxClient.list.callsArgWith(1, null, data) + return this.S3PersistorManager.directorySize( + this.bucketName, + this.key, + (err, totalSize) => { + totalSize.should.equal(3072) + return done() + } + ) + })) +}) diff --git a/services/filestore/test/unit/js/SafeExecTests.js b/services/filestore/test/unit/js/SafeExecTests.js index f9a0e51ca8..2b629947f5 100644 --- a/services/filestore/test/unit/js/SafeExecTests.js +++ b/services/filestore/test/unit/js/SafeExecTests.js @@ -9,73 +9,87 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const { - assert -} = require("chai"); -const sinon = require('sinon'); -const chai = require('chai'); -const should = chai.should(); -const { - expect -} = chai; -const modulePath = "../../../app/js/SafeExec.js"; -const SandboxedModule = require('sandboxed-module'); +const { assert } = require('chai') +const sinon = require('sinon') +const chai = require('chai') +const should = chai.should() +const { expect } = chai +const modulePath = '../../../app/js/SafeExec.js' +const SandboxedModule = require('sandboxed-module') -describe("SafeExec", function() { +describe('SafeExec', function() { + beforeEach(function() { + this.settings = { enableConversions: true } + this.safe_exec = SandboxedModule.require(modulePath, { + requires: { + 'logger-sharelatex': { + log() {}, + err() {} + }, + 'settings-sharelatex': this.settings + } + }) + return (this.options = { timeout: 10 * 1000, killSignal: 'SIGTERM' }) + }) - beforeEach(function() { - this.settings = - {enableConversions:true}; - this.safe_exec = SandboxedModule.require(modulePath, { requires: { - "logger-sharelatex": { - log() {}, - err() {} - }, - "settings-sharelatex": this.settings - } - } - ); - return this.options = {timeout: 10*1000, killSignal: "SIGTERM" };}); + return describe('safe_exec', function() { + it('should execute a valid command', function(done) { + return this.safe_exec( + ['/bin/echo', 'hello'], + this.options, + (err, stdout, stderr) => { + stdout.should.equal('hello\n') + should.not.exist(err) + return done() + } + ) + }) - return describe("safe_exec", function() { + it('should error when conversions are disabled', function(done) { + this.settings.enableConversions = false + return this.safe_exec( + ['/bin/echo', 'hello'], + this.options, + (err, stdout, stderr) => { + expect(err).to.exist + return done() + } + ) + }) - it("should execute a valid command", function(done) { - return this.safe_exec(["/bin/echo", "hello"], this.options, (err, stdout, stderr) => { - stdout.should.equal("hello\n"); - should.not.exist(err); - return done(); - }); - }); + it('should execute a command with non-zero exit status', function(done) { + return this.safe_exec( + ['/usr/bin/env', 'false'], + this.options, + (err, stdout, stderr) => { + stdout.should.equal('') + stderr.should.equal('') + err.message.should.equal('exit status 1') + return done() + } + ) + }) - it("should error when conversions are disabled", function(done) { - this.settings.enableConversions = false; - return this.safe_exec(["/bin/echo", "hello"], this.options, (err, stdout, stderr) => { - expect(err).to.exist; - return done(); - }); - }); + it('should handle an invalid command', function(done) { + return this.safe_exec( + ['/bin/foobar'], + this.options, + (err, stdout, stderr) => { + err.code.should.equal('ENOENT') + return done() + } + ) + }) - it("should execute a command with non-zero exit status", function(done) { - return this.safe_exec(["/usr/bin/env", "false"], this.options, (err, stdout, stderr) => { - stdout.should.equal(""); - stderr.should.equal(""); - err.message.should.equal("exit status 1"); - return done(); - }); - }); - - it("should handle an invalid command", function(done) { - return this.safe_exec(["/bin/foobar"], this.options, (err, stdout, stderr) => { - err.code.should.equal("ENOENT"); - return done(); - }); - }); - - return it("should handle a command that runs too long", function(done) { - return this.safe_exec(["/bin/sleep", "10"], {timeout: 500, killSignal: "SIGTERM"}, (err, stdout, stderr) => { - err.should.equal("SIGTERM"); - return done(); - }); - }); - }); -}); + return it('should handle a command that runs too long', function(done) { + return this.safe_exec( + ['/bin/sleep', '10'], + { timeout: 500, killSignal: 'SIGTERM' }, + (err, stdout, stderr) => { + err.should.equal('SIGTERM') + return done() + } + ) + }) + }) +}) diff --git a/services/filestore/test/unit/js/SettingsTests.js b/services/filestore/test/unit/js/SettingsTests.js index 98d17723a9..472c6d1179 100644 --- a/services/filestore/test/unit/js/SettingsTests.js +++ b/services/filestore/test/unit/js/SettingsTests.js @@ -9,27 +9,25 @@ * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ -const { - assert -} = require("chai"); -const sinon = require('sinon'); -const chai = require('chai'); -const should = chai.should(); -const { - expect -} = chai; -const modulePath = "../../../app/js/BucketController.js"; +const { assert } = require('chai') +const sinon = require('sinon') +const chai = require('chai') +const should = chai.should() +const { expect } = chai +const modulePath = '../../../app/js/BucketController.js' -describe("Settings", () => describe("s3", () => it("should use JSONified env var if present", function(done){ - const s3_settings = { +describe('Settings', () => + describe('s3', () => + it('should use JSONified env var if present', function(done) { + const s3_settings = { bucket1: { - auth_key: 'bucket1_key', - auth_secret: 'bucket1_secret' + auth_key: 'bucket1_key', + auth_secret: 'bucket1_secret' } - }; - process.env.S3_BUCKET_CREDENTIALS = JSON.stringify(s3_settings); + } + process.env.S3_BUCKET_CREDENTIALS = JSON.stringify(s3_settings) - const settings = require("settings-sharelatex"); - expect(settings.filestore.s3BucketCreds).to.deep.equal(s3_settings); - return done(); -}))); + const settings = require('settings-sharelatex') + expect(settings.filestore.s3BucketCreds).to.deep.equal(s3_settings) + return done() + }))) From 495424a94b0df234d8c9c6efcfc5326e239b0501 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 16 Dec 2019 11:28:24 +0000 Subject: [PATCH 27/73] Fix metrics includes in SandboxedModule --- services/filestore/test/unit/js/FileConverterTests.js | 6 ++---- services/filestore/test/unit/js/LocalFileWriterTests.js | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/services/filestore/test/unit/js/FileConverterTests.js b/services/filestore/test/unit/js/FileConverterTests.js index 1b548d592e..89faacd1d1 100644 --- a/services/filestore/test/unit/js/FileConverterTests.js +++ b/services/filestore/test/unit/js/FileConverterTests.js @@ -29,10 +29,8 @@ describe('FileConverter', function() { err() {} }, 'metrics-sharelatex': { - inc() {}, - Timer() { - return { done() {} } - } + inc: sinon.stub(), + Timer: sinon.stub().returns({ done: sinon.stub() }) }, 'settings-sharelatex': (this.Settings = { commands: { diff --git a/services/filestore/test/unit/js/LocalFileWriterTests.js b/services/filestore/test/unit/js/LocalFileWriterTests.js index e7ea993ab9..04cc2fb049 100644 --- a/services/filestore/test/unit/js/LocalFileWriterTests.js +++ b/services/filestore/test/unit/js/LocalFileWriterTests.js @@ -48,10 +48,8 @@ describe('LocalFileWriter', function() { }, 'settings-sharelatex': this.settings, 'metrics-sharelatex': { - inc() {}, - Timer() { - return { done() {} } - } + inc: sinon.stub(), + Timer: sinon.stub().returns({ done: sinon.stub() }) } } }) From 6974fd640d496a2d5ebc58d69fb1849b0a1436e6 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 16 Dec 2019 11:29:25 +0000 Subject: [PATCH 28/73] Remove unusued cluster.coffee --- services/filestore/cluster.coffee | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 services/filestore/cluster.coffee diff --git a/services/filestore/cluster.coffee b/services/filestore/cluster.coffee deleted file mode 100644 index a0ca60a619..0000000000 --- a/services/filestore/cluster.coffee +++ /dev/null @@ -1,9 +0,0 @@ -recluster = require "recluster" # https://github.com/doxout/recluster -path = require "path" - -cluster = recluster path.join(__dirname, 'app.js'), { - workers: 2, - backoff: 0, - readyWhen: "listening" -} -cluster.run() From 35d050a49ca8a10c5f0f06f417d1acc31f821fbc Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 16 Dec 2019 11:32:46 +0000 Subject: [PATCH 29/73] Prettier fixes --- services/filestore/test/unit/js/S3PersistorManagerTests.js | 1 - 1 file changed, 1 deletion(-) diff --git a/services/filestore/test/unit/js/S3PersistorManagerTests.js b/services/filestore/test/unit/js/S3PersistorManagerTests.js index c0a03696a8..777c9c5a32 100644 --- a/services/filestore/test/unit/js/S3PersistorManagerTests.js +++ b/services/filestore/test/unit/js/S3PersistorManagerTests.js @@ -317,7 +317,6 @@ describe('S3PersistorManagerTests', function() { } ) }) - ;[403, 404].forEach(statusCode => it(`should throw NotFoundError when S3 responds with ${statusCode}`, function(done) { const error = new Error() From 230baef8fa3a8d6ae3814cc8a5854bf2f598ca9a Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 16 Dec 2019 11:32:58 +0000 Subject: [PATCH 30/73] Add linting tasks to package.json --- services/filestore/package.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/filestore/package.json b/services/filestore/package.json index 40aabd8279..24ec708b10 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -13,6 +13,9 @@ "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", "start": "node $NODE_APP_OPTIONS app.js", "nodemon": "nodemon --config nodemon.json", + "lint": "eslint -f unix .", + "format": "prettier-eslint \"$(pwd)/**/*.js\" --list-different", + "format:fix": "prettier-eslint \"$(pwd)/**/*.js\" --write", "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js", "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js" }, From 7870c8185ecdc0265eac89df7d45d876832367b5 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 16 Dec 2019 11:58:22 +0000 Subject: [PATCH 31/73] Add linting to makefile --- services/filestore/.prettierignore | 1 + services/filestore/Makefile | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 services/filestore/.prettierignore diff --git a/services/filestore/.prettierignore b/services/filestore/.prettierignore new file mode 100644 index 0000000000..3c3629e647 --- /dev/null +++ b/services/filestore/.prettierignore @@ -0,0 +1 @@ +node_modules diff --git a/services/filestore/Makefile b/services/filestore/Makefile index e83a0696e0..b5b188aa51 100644 --- a/services/filestore/Makefile +++ b/services/filestore/Makefile @@ -21,7 +21,17 @@ clean: rm -rf test/unit/js rm -rf test/acceptance/js -test: test_unit test_acceptance +format: + $(DOCKER_COMPOSE) run --rm test_unit npm run format + +format_fix: + $(DOCKER_COMPOSE) run --rm test_unit npm run format:fix + +lint: + $(DOCKER_COMPOSE) run --rm test_unit npm run lint + + +test: format lint test_unit test_acceptance test_unit: @[ ! -d test/unit ] && echo "filestore has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit From 29f333d60cac7fe4cab99c0ba4fc763f47898190 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 16 Dec 2019 12:14:06 +0000 Subject: [PATCH 32/73] Placate eslint --- services/filestore/app.js | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/services/filestore/app.js b/services/filestore/app.js index cbe13e8076..c6c11e152a 100644 --- a/services/filestore/app.js +++ b/services/filestore/app.js @@ -4,6 +4,7 @@ * DS207: Consider shorter variations of null checks * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ +/* eslint-disable node/no-deprecated-api */ const Metrics = require('metrics-sharelatex') Metrics.initialize('filestore') const express = require('express') @@ -11,7 +12,6 @@ const bodyParser = require('body-parser') let logger = require('logger-sharelatex') logger.initialize('filestore') const settings = require('settings-sharelatex') -const request = require('request') const fileController = require('./app/js/FileController') const bucketController = require('./app/js/BucketController') const keyBuilder = require('./app/js/KeyBuilder') @@ -170,10 +170,15 @@ app.get( app.get('/bucket/:bucket/key/*', bucketController.getFile) -app.get('/heapdump', (req, res) => +app.get('/heapdump', (req, res, next) => require('heapdump').writeSnapshot( '/tmp/' + Date.now() + '.filestore.heapsnapshot', - (err, filename) => res.send(filename) + (err, filename) => { + if (err) { + return next(err) + } + res.send(filename) + } ) ) @@ -219,9 +224,13 @@ const host = '0.0.0.0' if (!module.parent) { // Called directly - var server = app.listen(port, host, error => + var server = app.listen(port, host, error => { + if (error) { + logger.error('Error starting Filestore', error) + throw error + } logger.info(`Filestore starting up, listening on ${host}:${port}`) - ) + }) } module.exports = app @@ -232,10 +241,10 @@ process.on('SIGTERM', function() { }) if (global.gc != null) { - let oneMinute + const oneMinute = 60 * 1000 const gcTimer = setInterval(function() { global.gc() return logger.log(process.memoryUsage(), 'global.gc') - }, 3 * (oneMinute = 60 * 1000)) + }, 3 * oneMinute) gcTimer.unref() } From b7e098208f3b8befc3d28b723c9a584e4f1df3f9 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 16 Dec 2019 17:09:38 +0000 Subject: [PATCH 33/73] Update build files from build_scripts --- services/filestore/.eslintrc | 11 +- services/filestore/.nvmrc | 2 +- services/filestore/.prettierrc | 4 + services/filestore/Dockerfile | 11 +- services/filestore/Jenkinsfile | 8 + services/filestore/Makefile | 9 +- services/filestore/buildscript.txt | 16 +- .../filestore/config/settings.defaults.coffee | 6 +- services/filestore/docker-compose.ci.yml | 18 +- services/filestore/docker-compose.yml | 26 +- services/filestore/nodemon.json | 7 +- services/filestore/npm-shrinkwrap.json | 261 ++++++++---------- services/filestore/package.json | 12 +- 13 files changed, 177 insertions(+), 214 deletions(-) diff --git a/services/filestore/.eslintrc b/services/filestore/.eslintrc index f8776c68a0..e65cee6619 100644 --- a/services/filestore/.eslintrc +++ b/services/filestore/.eslintrc @@ -1,16 +1,23 @@ +// this file was auto-generated, do not edit it directly. +// instead run bin/update_build_scripts from +// https://github.com/sharelatex/sharelatex-dev-environment +// Version: 1.3 { "extends": [ "standard", "prettier", - "prettier/standard", + "prettier/standard" ], + "parserOptions": { + "ecmaVersion": 6 + }, "plugins": [ "mocha", "chai-expect", "chai-friendly" ], - "parser": "babel-eslint", "env": { + "node": true, "mocha": true }, "rules": { diff --git a/services/filestore/.nvmrc b/services/filestore/.nvmrc index e1e5d1369a..73bffb0393 100644 --- a/services/filestore/.nvmrc +++ b/services/filestore/.nvmrc @@ -1 +1 @@ -6.9.5 +10.17.0 diff --git a/services/filestore/.prettierrc b/services/filestore/.prettierrc index b2095be81e..ddf06f31b6 100644 --- a/services/filestore/.prettierrc +++ b/services/filestore/.prettierrc @@ -1,3 +1,7 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +# Version: 1.3 { "semi": false, "singleQuote": true diff --git a/services/filestore/Dockerfile b/services/filestore/Dockerfile index 5a18b41699..4cdf40264f 100644 --- a/services/filestore/Dockerfile +++ b/services/filestore/Dockerfile @@ -1,4 +1,9 @@ -FROM node:10.17.0-jessie as app +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/sharelatex/sharelatex-dev-environment +# Version: 1.3 + +FROM node:10.17.0 as app WORKDIR /app @@ -9,9 +14,9 @@ RUN npm install --quiet COPY . /app -RUN npm run compile:all -FROM node:10.17.0-jessie + +FROM node:10.17.0 COPY --from=app /app /app diff --git a/services/filestore/Jenkinsfile b/services/filestore/Jenkinsfile index dd741ce239..fe20b5e06e 100644 --- a/services/filestore/Jenkinsfile +++ b/services/filestore/Jenkinsfile @@ -16,6 +16,7 @@ pipeline { } stages { + stage('Install') { steps { withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) { @@ -36,6 +37,13 @@ pipeline { } } + stage('Linting') { + steps { + sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make format' + sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make lint' + } + } + stage('Unit Tests') { steps { sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit' diff --git a/services/filestore/Makefile b/services/filestore/Makefile index b5b188aa51..67bec94e5a 100644 --- a/services/filestore/Makefile +++ b/services/filestore/Makefile @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.24 +# Version: 1.3 BUILD_NUMBER ?= local BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) @@ -16,10 +16,6 @@ DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ clean: docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) - rm -f app.js - rm -rf app/js - rm -rf test/unit/js - rm -rf test/acceptance/js format: $(DOCKER_COMPOSE) run --rm test_unit npm run format @@ -30,7 +26,6 @@ format_fix: lint: $(DOCKER_COMPOSE) run --rm test_unit npm run lint - test: format lint test_unit test_acceptance test_unit: @@ -46,6 +41,7 @@ test_clean: test_acceptance_pre_run: @[ ! -f test/acceptance/js/scripts/pre-run ] && echo "filestore has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/js/scripts/pre-run + build: docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ --tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ @@ -58,4 +54,5 @@ publish: docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + .PHONY: clean test test_unit test_acceptance test_clean build publish diff --git a/services/filestore/buildscript.txt b/services/filestore/buildscript.txt index dc8c156383..773949832d 100644 --- a/services/filestore/buildscript.txt +++ b/services/filestore/buildscript.txt @@ -1,10 +1,10 @@ filestore ---language=coffeescript ---node-version=6.9.5 ---acceptance-creds=None ---dependencies=mongo,redis ---docker-repos=gcr.io/overleaf-ops ---build-target=docker ---script-version=1.1.24 ---env-pass-through= +--acceptance-creds= --public-repo=True +--language=es +--node-version=10.17.0 +--dependencies=mongo,redis,s3 +--script-version=1.3 +--docker-repos=gcr.io/overleaf-ops +--env-add=ENABLE_CONVERSIONS="true",USE_PROM_METRICS="true",AWS_S3_USER_FILES_BUCKET_NAME=fake_user_files,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake_template_files,AWS_S3_PUBLIC_FILES_BUCKET_NAME=fake_public_files,AWS_S3_ENDPOINT=http://s3:9090,AWS_ACCESS_KEY_ID=fake,AWS_SECRET_ACCESS_KEY=fake +--env-pass-through= diff --git a/services/filestore/config/settings.defaults.coffee b/services/filestore/config/settings.defaults.coffee index 29010e85ee..95256a33d6 100644 --- a/services/filestore/config/settings.defaults.coffee +++ b/services/filestore/config/settings.defaults.coffee @@ -11,11 +11,11 @@ settings = # Choices are # s3 - Amazon S3 # fs - local filesystem - if process.env['AWS_KEY']? or process.env['S3_BUCKET_CREDENTIALS']? + if process.env['AWS_ACCESS_KEY_ID']? or process.env['S3_BUCKET_CREDENTIALS']? backend: "s3" s3: - key: process.env['AWS_KEY'] - secret: process.env['AWS_SECRET'] + key: process.env['AWS_ACCESS_KEY_ID'] + secret: process.env['AWS_SECRET_ACCESS_KEY'] endpoint: process.env['AWS_S3_ENDPOINT'] stores: user_files: process.env['AWS_S3_USER_FILES_BUCKET_NAME'] diff --git a/services/filestore/docker-compose.ci.yml b/services/filestore/docker-compose.ci.yml index 42c6ae37b5..d8342a9837 100644 --- a/services/filestore/docker-compose.ci.yml +++ b/services/filestore/docker-compose.ci.yml @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.24 +# Version: 1.3 version: "2.1" @@ -22,28 +22,27 @@ services: REDIS_HOST: redis MONGO_HOST: mongo POSTGRES_HOST: postgres - ENABLE_CONVERSIONS: "true" MOCHA_GREP: ${MOCHA_GREP} NODE_ENV: test + ENABLE_CONVERSIONS: "true" USE_PROM_METRICS: "true" - AWS_KEY: fake - AWS_SECRET: fake AWS_S3_USER_FILES_BUCKET_NAME: fake_user_files AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake_template_files AWS_S3_PUBLIC_FILES_BUCKET_NAME: fake_public_files - AWS_S3_ENDPOINT: http://fakes3:9090 + AWS_S3_ENDPOINT: http://s3:9090 + AWS_ACCESS_KEY_ID: fake + AWS_SECRET_ACCESS_KEY: fake depends_on: mongo: condition: service_healthy redis: condition: service_healthy - fakes3: + s3: condition: service_healthy user: node command: npm run test:acceptance:_run - tar: build: . image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER @@ -51,17 +50,14 @@ services: - ./:/tmp/build/ command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . user: root - redis: image: redis mongo: image: mongo:3.4 - - fakes3: + s3: image: adobe/s3mock environment: - initialBuckets=fake_user_files,fake_template_files,fake_public_files healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9090"] - diff --git a/services/filestore/docker-compose.yml b/services/filestore/docker-compose.yml index 65f18f4d78..1269e795a2 100644 --- a/services/filestore/docker-compose.yml +++ b/services/filestore/docker-compose.yml @@ -1,7 +1,7 @@ # This file was auto-generated, do not edit it directly. # Instead run bin/update_build_scripts from # https://github.com/sharelatex/sharelatex-dev-environment -# Version: 1.1.24 +# Version: 1.3 version: "2.1" @@ -28,47 +28,35 @@ services: MONGO_HOST: mongo POSTGRES_HOST: postgres MOCHA_GREP: ${MOCHA_GREP} - ENABLE_CONVERSIONS: "true" LOG_LEVEL: ERROR NODE_ENV: test + ENABLE_CONVERSIONS: "true" USE_PROM_METRICS: "true" - AWS_KEY: fake - AWS_SECRET: fake AWS_S3_USER_FILES_BUCKET_NAME: fake_user_files AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake_template_files AWS_S3_PUBLIC_FILES_BUCKET_NAME: fake_public_files - AWS_S3_ENDPOINT: http://fakes3:9090 + AWS_S3_ENDPOINT: http://s3:9090 + AWS_ACCESS_KEY_ID: fake + AWS_SECRET_ACCESS_KEY: fake user: node depends_on: mongo: condition: service_healthy redis: condition: service_healthy - fakes3: + s3: condition: service_healthy command: npm run test:acceptance - - - tar: - build: . - image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER - volumes: - - ./:/tmp/build/ - command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . - user: root - redis: image: redis mongo: image: mongo:3.4 - fakes3: + s3: image: adobe/s3mock environment: - initialBuckets=fake_user_files,fake_template_files,fake_public_files healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9090"] - - diff --git a/services/filestore/nodemon.json b/services/filestore/nodemon.json index 98db38d71b..5826281b84 100644 --- a/services/filestore/nodemon.json +++ b/services/filestore/nodemon.json @@ -10,10 +10,9 @@ }, "watch": [ - "app/coffee/", - "app.coffee", + "app/js/", + "app.js", "config/" ], - "ext": "coffee" - + "ext": "js" } diff --git a/services/filestore/npm-shrinkwrap.json b/services/filestore/npm-shrinkwrap.json index 64529d81ca..d636bbe990 100644 --- a/services/filestore/npm-shrinkwrap.json +++ b/services/filestore/npm-shrinkwrap.json @@ -646,18 +646,15 @@ } }, "ansi-escapes": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.0.tgz", - "integrity": "sha512-EiYhwo0v255HUL6eDyuLrXEkTi7WwVCLAw+SeOQ7M7qdun1z1pum4DEm/nuqIVbPvi9RPPc9k9LbyBv6H0DwVg==", - "dev": true, - "requires": { - "type-fest": "^0.8.1" - } + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz", + "integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==", + "dev": true }, "ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", "dev": true }, "ansi-styles": { @@ -1036,12 +1033,12 @@ "dev": true }, "cli-cursor": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", - "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", + "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=", "dev": true, "requires": { - "restore-cursor": "^3.1.0" + "restore-cursor": "^2.0.0" } }, "cli-width": { @@ -1372,12 +1369,6 @@ "shimmer": "^1.2.0" } }, - "emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true - }, "encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -1459,48 +1450,47 @@ "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" }, "eslint": { - "version": "6.7.2", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-6.7.2.tgz", - "integrity": "sha512-qMlSWJaCSxDFr8fBPvJM9kJwbazrhNcBU3+DszDW1OlEwKBBRWsJc7NJFelvwQpanHCR14cOLD41x8Eqvo3Nng==", + "version": "5.16.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-5.16.0.tgz", + "integrity": "sha512-S3Rz11i7c8AA5JPv7xAH+dOyq/Cu/VXHiHXBPOU1k/JAM5dXqQPt3qcrhpHSorXmrpu2g0gkIBVXAqCpzfoZIg==", "dev": true, "requires": { "@babel/code-frame": "^7.0.0", - "ajv": "^6.10.0", + "ajv": "^6.9.1", "chalk": "^2.1.0", "cross-spawn": "^6.0.5", "debug": "^4.0.1", "doctrine": "^3.0.0", - "eslint-scope": "^5.0.0", - "eslint-utils": "^1.4.3", - "eslint-visitor-keys": "^1.1.0", - "espree": "^6.1.2", + "eslint-scope": "^4.0.3", + "eslint-utils": "^1.3.1", + "eslint-visitor-keys": "^1.0.0", + "espree": "^5.0.1", "esquery": "^1.0.1", "esutils": "^2.0.2", "file-entry-cache": "^5.0.1", "functional-red-black-tree": "^1.0.1", - "glob-parent": "^5.0.0", - "globals": "^12.1.0", + "glob": "^7.1.2", + "globals": "^11.7.0", "ignore": "^4.0.6", "import-fresh": "^3.0.0", "imurmurhash": "^0.1.4", - "inquirer": "^7.0.0", - "is-glob": "^4.0.0", - "js-yaml": "^3.13.1", + "inquirer": "^6.2.2", + "js-yaml": "^3.13.0", "json-stable-stringify-without-jsonify": "^1.0.1", "levn": "^0.3.0", - "lodash": "^4.17.14", + "lodash": "^4.17.11", "minimatch": "^3.0.4", "mkdirp": "^0.5.1", "natural-compare": "^1.4.0", - "optionator": "^0.8.3", + "optionator": "^0.8.2", + "path-is-inside": "^1.0.2", "progress": "^2.0.0", "regexpp": "^2.0.1", - "semver": "^6.1.2", - "strip-ansi": "^5.2.0", - "strip-json-comments": "^3.0.1", + "semver": "^5.5.1", + "strip-ansi": "^4.0.0", + "strip-json-comments": "^2.0.1", "table": "^5.2.3", - "text-table": "^0.2.0", - "v8-compile-cache": "^2.0.3" + "text-table": "^0.2.0" }, "dependencies": { "debug": { @@ -1512,11 +1502,19 @@ "ms": "^2.1.1" } }, - "lodash": { - "version": "4.17.15", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", - "dev": true + "glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } }, "ms": { "version": "2.1.2", @@ -1525,10 +1523,19 @@ "dev": true }, "semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", "dev": true + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "dev": true, + "requires": { + "ansi-regex": "^3.0.0" + } } } }, @@ -1691,9 +1698,9 @@ "dev": true }, "eslint-scope": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.0.0.tgz", - "integrity": "sha512-oYrhJW7S0bxAFDvWqzvMPRm6pcgcnWc4QnofCAqRTRfQC0JcwenzGglTtsLyIuuWFfkqDG9vz67cnttSd53djw==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", + "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==", "dev": true, "requires": { "esrecurse": "^4.1.0", @@ -1716,22 +1723,14 @@ "dev": true }, "espree": { - "version": "6.1.2", - "resolved": "https://registry.npmjs.org/espree/-/espree-6.1.2.tgz", - "integrity": "sha512-2iUPuuPP+yW1PZaMSDM9eyVf8D5P0Hi8h83YtZ5bPc/zHYjII5khoixIUTMO794NOY8F/ThF1Bo8ncZILarUTA==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-5.0.1.tgz", + "integrity": "sha512-qWAZcWh4XE/RwzLJejfcofscgMc9CamR6Tn1+XRXNzrvUSSbiAjGOI/fggztjIi7y9VLPqnICMIPiGyr8JaZ0A==", "dev": true, "requires": { - "acorn": "^7.1.0", - "acorn-jsx": "^5.1.0", - "eslint-visitor-keys": "^1.1.0" - }, - "dependencies": { - "acorn": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.1.0.tgz", - "integrity": "sha512-kL5CuoXA/dgxlBbVrflsflzQ3PAas7RYZB52NOm/6839iVYJgKMJ3cQJD+t2i5+qFa8h3MDpEOJiS64E8JLnSQ==", - "dev": true - } + "acorn": "^6.0.7", + "acorn-jsx": "^5.0.0", + "eslint-visitor-keys": "^1.0.0" } }, "esprima": { @@ -1950,9 +1949,9 @@ "integrity": "sha1-PlzoKTQJz6pxd6cbnKhOGx5vJe8=" }, "figures": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/figures/-/figures-3.1.0.tgz", - "integrity": "sha512-ravh8VRXqHuMvZt/d8GblBeqDMkdJMBdv/2KntFH+ra5MXkO7nxNKpzQ3n6QD/2da1kH0aWmNISdvhM7gl2gVg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", + "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=", "dev": true, "requires": { "escape-string-regexp": "^1.0.5" @@ -2190,23 +2189,11 @@ "path-is-absolute": "^1.0.0" } }, - "glob-parent": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.0.tgz", - "integrity": "sha512-qjtRgnIVmOfnKUE3NJAQEdk+lKrxfw8t5ke7SXtfMTHcjsBfOfWXCQfdb30zfDoZQ2IRSIiidmjtbHZPZ++Ihw==", - "dev": true, - "requires": { - "is-glob": "^4.0.1" - } - }, "globals": { - "version": "12.3.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-12.3.0.tgz", - "integrity": "sha512-wAfjdLgFsPZsklLJvOBUBmzYE8/CwhEqSBEMRXA3qxIiNtyqvjYurAtIfDh6chlEPUfmTY3MnZh5Hfh4q0UlIw==", - "dev": true, - "requires": { - "type-fest": "^0.8.1" - } + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "dev": true }, "google-auth-library": { "version": "3.1.2", @@ -2453,22 +2440,22 @@ "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, "inquirer": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-7.0.1.tgz", - "integrity": "sha512-V1FFQ3TIO15det8PijPLFR9M9baSlnRs9nL7zWu1MNVA2T9YVl9ZbrHJhYs7e9X8jeMZ3lr2JH/rdHFgNCBdYw==", + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.5.2.tgz", + "integrity": "sha512-cntlB5ghuB0iuO65Ovoi8ogLHiWGs/5yNrtUcKjFhSSiVeAIVpD7koaSU9RM8mpXw5YDi9RdYXGQMaOURB7ycQ==", "dev": true, "requires": { - "ansi-escapes": "^4.2.1", + "ansi-escapes": "^3.2.0", "chalk": "^2.4.2", - "cli-cursor": "^3.1.0", + "cli-cursor": "^2.1.0", "cli-width": "^2.0.0", "external-editor": "^3.0.3", - "figures": "^3.0.0", - "lodash": "^4.17.15", - "mute-stream": "0.0.8", + "figures": "^2.0.0", + "lodash": "^4.17.12", + "mute-stream": "0.0.7", "run-async": "^2.2.0", - "rxjs": "^6.5.3", - "string-width": "^4.1.0", + "rxjs": "^6.4.0", + "string-width": "^2.1.0", "strip-ansi": "^5.1.0", "through": "^2.3.6" }, @@ -2514,27 +2501,12 @@ "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY=", "dev": true }, - "is-extglob": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", - "dev": true - }, "is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", "dev": true }, - "is-glob": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz", - "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==", - "dev": true, - "requires": { - "is-extglob": "^2.1.1" - } - }, "is-promise": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz", @@ -3103,9 +3075,9 @@ } }, "mimic-fn": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", - "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", + "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", "dev": true }, "minimatch": { @@ -3219,9 +3191,9 @@ "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, "mute-stream": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", - "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", + "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=", "dev": true }, "mv": { @@ -3462,12 +3434,12 @@ } }, "onetime": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.0.tgz", - "integrity": "sha512-5NcSkPHhwTVFIQN+TUqXoS5+dlElHXdpAWu9I0HP20YOtIi+aZ0Ct82jdlILDxjLEAWwvm+qj1m6aEtsDVmm6Q==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", + "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=", "dev": true, "requires": { - "mimic-fn": "^2.1.0" + "mimic-fn": "^1.0.0" } }, "optionator": { @@ -4582,12 +4554,12 @@ } }, "restore-cursor": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", - "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", + "integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=", "dev": true, "requires": { - "onetime": "^5.1.0", + "onetime": "^2.0.0", "signal-exit": "^3.0.2" } }, @@ -4928,23 +4900,22 @@ "integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI=" }, "string-width": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz", - "integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", "dev": true, "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.0" + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" }, "dependencies": { "strip-ansi": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", - "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", "dev": true, "requires": { - "ansi-regex": "^5.0.0" + "ansi-regex": "^3.0.0" } } } @@ -4998,9 +4969,9 @@ "dev": true }, "strip-json-comments": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.0.1.tgz", - "integrity": "sha512-VTyMAUfdm047mwKl+u79WIdrZxtFtn+nBxHeb844XBQ9uMNTuTHdx2hc5RiAJYqwTj3wc/xe5HLSdJSkJ+WfZw==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", "dev": true }, "supports-color": { @@ -5198,12 +5169,6 @@ "integrity": "sha1-dkb7XxiHHPu3dJ5pvTmmOI63RQw=", "dev": true }, - "type-fest": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", - "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", - "dev": true - }, "type-is": { "version": "1.6.16", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz", @@ -5263,12 +5228,6 @@ "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, - "v8-compile-cache": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.0.tgz", - "integrity": "sha512-usZBT3PW+LOjM25wbqIlZwPeJV+3OSz3M1k1Ws8snlW39dZyYL9lOGC5FgPVHfk0jKmjiDV8Z0mIbVQPiwFs7g==", - "dev": true - }, "validate-npm-package-license": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", diff --git a/services/filestore/package.json b/services/filestore/package.json index 24ec708b10..f9bba8a746 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -13,11 +13,11 @@ "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP", "start": "node $NODE_APP_OPTIONS app.js", "nodemon": "nodemon --config nodemon.json", - "lint": "eslint -f unix .", - "format": "prettier-eslint \"$(pwd)/**/*.js\" --list-different", - "format:fix": "prettier-eslint \"$(pwd)/**/*.js\" --write", - "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js", - "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js" + "lint": "node_modules/.bin/eslint .", + "format": "node_modules/.bin/prettier-eslint '**/*.js' --list-different", + "format:fix": "node_modules/.bin/prettier-eslint '**/*.js' --write", + "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js", + "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js" }, "dependencies": { "async": "~0.2.10", @@ -47,7 +47,7 @@ "babel-eslint": "^10.0.3", "bunyan": "^1.3.5", "chai": "4.2.0", - "eslint": "^6.7.2", + "eslint": "^5.16.0", "eslint-config-prettier": "^6.7.0", "eslint-config-standard": "^14.1.0", "eslint-plugin-chai-expect": "^2.1.0", From 0b3b71b857d2fb6a2b3483916f52e9fa92eda7c8 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Tue, 17 Dec 2019 09:57:51 +0000 Subject: [PATCH 34/73] Fall back to old AWS env vars if present --- services/filestore/config/settings.defaults.coffee | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/services/filestore/config/settings.defaults.coffee b/services/filestore/config/settings.defaults.coffee index 95256a33d6..206f932a76 100644 --- a/services/filestore/config/settings.defaults.coffee +++ b/services/filestore/config/settings.defaults.coffee @@ -1,5 +1,12 @@ Path = require "path" +# environment variables renamed for consistency +# use AWS_ACCESS_KEY_ID-style going forward +if process.env['AWS_KEY'] && !process.env['AWS_ACCESS_KEY_ID'] + process.env['AWS_ACCESS_KEY_ID'] = process.env['AWS_KEY'] +if process.env['AWS_SECRET'] && !process.env['AWS_SECRET_ACCESS_KEY'] + process.env['AWS_SECRET_ACCESS_KEY'] = process.env['AWS_SECRET'] + settings = internal: filestore: From 42f6d1003147d2df00e852b0b0424a5b41599279 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Tue, 17 Dec 2019 15:19:19 +0000 Subject: [PATCH 35/73] Remove mongo and redis from dependencies --- services/filestore/buildscript.txt | 14 +++++++------- services/filestore/docker-compose.ci.yml | 9 --------- services/filestore/docker-compose.yml | 10 ---------- 3 files changed, 7 insertions(+), 26 deletions(-) diff --git a/services/filestore/buildscript.txt b/services/filestore/buildscript.txt index 773949832d..7e00306b70 100644 --- a/services/filestore/buildscript.txt +++ b/services/filestore/buildscript.txt @@ -1,10 +1,10 @@ filestore ---acceptance-creds= ---public-repo=True ---language=es ---node-version=10.17.0 ---dependencies=mongo,redis,s3 ---script-version=1.3 ---docker-repos=gcr.io/overleaf-ops --env-add=ENABLE_CONVERSIONS="true",USE_PROM_METRICS="true",AWS_S3_USER_FILES_BUCKET_NAME=fake_user_files,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake_template_files,AWS_S3_PUBLIC_FILES_BUCKET_NAME=fake_public_files,AWS_S3_ENDPOINT=http://s3:9090,AWS_ACCESS_KEY_ID=fake,AWS_SECRET_ACCESS_KEY=fake +--dependencies=s3 +--acceptance-creds= +--script-version=1.3 +--node-version=10.17.0 +--language=es +--docker-repos=gcr.io/overleaf-ops +--public-repo=True --env-pass-through= diff --git a/services/filestore/docker-compose.ci.yml b/services/filestore/docker-compose.ci.yml index d8342a9837..ae9b43ee25 100644 --- a/services/filestore/docker-compose.ci.yml +++ b/services/filestore/docker-compose.ci.yml @@ -33,10 +33,6 @@ services: AWS_ACCESS_KEY_ID: fake AWS_SECRET_ACCESS_KEY: fake depends_on: - mongo: - condition: service_healthy - redis: - condition: service_healthy s3: condition: service_healthy user: node @@ -50,11 +46,6 @@ services: - ./:/tmp/build/ command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . user: root - redis: - image: redis - - mongo: - image: mongo:3.4 s3: image: adobe/s3mock environment: diff --git a/services/filestore/docker-compose.yml b/services/filestore/docker-compose.yml index 1269e795a2..792d70800b 100644 --- a/services/filestore/docker-compose.yml +++ b/services/filestore/docker-compose.yml @@ -40,20 +40,10 @@ services: AWS_SECRET_ACCESS_KEY: fake user: node depends_on: - mongo: - condition: service_healthy - redis: - condition: service_healthy s3: condition: service_healthy command: npm run test:acceptance - redis: - image: redis - - mongo: - image: mongo:3.4 - s3: image: adobe/s3mock environment: From b94df9952ba88e56ac4847a18e11401b60071867 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Wed, 18 Dec 2019 09:34:25 +0000 Subject: [PATCH 36/73] Add OError --- services/filestore/npm-shrinkwrap.json | 5 +++++ services/filestore/package.json | 1 + 2 files changed, 6 insertions(+) diff --git a/services/filestore/npm-shrinkwrap.json b/services/filestore/npm-shrinkwrap.json index d636bbe990..fb1ada1128 100644 --- a/services/filestore/npm-shrinkwrap.json +++ b/services/filestore/npm-shrinkwrap.json @@ -368,6 +368,11 @@ } } }, + "@overleaf/o-error": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@overleaf/o-error/-/o-error-2.1.0.tgz", + "integrity": "sha512-Zd9sks9LrLw8ErHt/cXeWIkyxWAqNAvNGn7wIjLQJH6TTEEW835PWOhpch+hQwwWsTxWIx/JDj+IpZ3ouw925g==" + }, "@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", diff --git a/services/filestore/package.json b/services/filestore/package.json index f9bba8a746..47aa4d0d96 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -20,6 +20,7 @@ "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js" }, "dependencies": { + "@overleaf/o-error": "^2.1.0", "async": "~0.2.10", "aws-sdk": "^2.1.39", "body-parser": "^1.2.0", From f19d82369d91f06cbc4f0f80b532fd0ea6cc408d Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Wed, 18 Dec 2019 09:52:18 +0000 Subject: [PATCH 37/73] Bump eslintrc up again to last-known-compatible version with eslit-prettier --- services/filestore/.eslintrc | 2 +- services/filestore/npm-shrinkwrap.json | 130 +++++++++++++++---------- services/filestore/package.json | 2 +- 3 files changed, 78 insertions(+), 56 deletions(-) diff --git a/services/filestore/.eslintrc b/services/filestore/.eslintrc index e65cee6619..6bd234a2dc 100644 --- a/services/filestore/.eslintrc +++ b/services/filestore/.eslintrc @@ -9,7 +9,7 @@ "prettier/standard" ], "parserOptions": { - "ecmaVersion": 6 + "ecmaVersion": 2017 }, "plugins": [ "mocha", diff --git a/services/filestore/npm-shrinkwrap.json b/services/filestore/npm-shrinkwrap.json index fb1ada1128..ef0f78fc15 100644 --- a/services/filestore/npm-shrinkwrap.json +++ b/services/filestore/npm-shrinkwrap.json @@ -1455,47 +1455,48 @@ "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" }, "eslint": { - "version": "5.16.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-5.16.0.tgz", - "integrity": "sha512-S3Rz11i7c8AA5JPv7xAH+dOyq/Cu/VXHiHXBPOU1k/JAM5dXqQPt3qcrhpHSorXmrpu2g0gkIBVXAqCpzfoZIg==", + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-6.4.0.tgz", + "integrity": "sha512-WTVEzK3lSFoXUovDHEbkJqCVPEPwbhCq4trDktNI6ygs7aO41d4cDT0JFAT5MivzZeVLWlg7vHL+bgrQv/t3vA==", "dev": true, "requires": { "@babel/code-frame": "^7.0.0", - "ajv": "^6.9.1", + "ajv": "^6.10.0", "chalk": "^2.1.0", "cross-spawn": "^6.0.5", "debug": "^4.0.1", "doctrine": "^3.0.0", - "eslint-scope": "^4.0.3", - "eslint-utils": "^1.3.1", - "eslint-visitor-keys": "^1.0.0", - "espree": "^5.0.1", + "eslint-scope": "^5.0.0", + "eslint-utils": "^1.4.2", + "eslint-visitor-keys": "^1.1.0", + "espree": "^6.1.1", "esquery": "^1.0.1", "esutils": "^2.0.2", "file-entry-cache": "^5.0.1", "functional-red-black-tree": "^1.0.1", - "glob": "^7.1.2", + "glob-parent": "^5.0.0", "globals": "^11.7.0", "ignore": "^4.0.6", "import-fresh": "^3.0.0", "imurmurhash": "^0.1.4", - "inquirer": "^6.2.2", - "js-yaml": "^3.13.0", + "inquirer": "^6.4.1", + "is-glob": "^4.0.0", + "js-yaml": "^3.13.1", "json-stable-stringify-without-jsonify": "^1.0.1", "levn": "^0.3.0", - "lodash": "^4.17.11", + "lodash": "^4.17.14", "minimatch": "^3.0.4", "mkdirp": "^0.5.1", "natural-compare": "^1.4.0", "optionator": "^0.8.2", - "path-is-inside": "^1.0.2", "progress": "^2.0.0", "regexpp": "^2.0.1", - "semver": "^5.5.1", - "strip-ansi": "^4.0.0", - "strip-json-comments": "^2.0.1", + "semver": "^6.1.2", + "strip-ansi": "^5.2.0", + "strip-json-comments": "^3.0.1", "table": "^5.2.3", - "text-table": "^0.2.0" + "text-table": "^0.2.0", + "v8-compile-cache": "^2.0.3" }, "dependencies": { "debug": { @@ -1507,19 +1508,11 @@ "ms": "^2.1.1" } }, - "glob": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", - "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", - "dev": true, - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } + "lodash": { + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", + "dev": true }, "ms": { "version": "2.1.2", @@ -1528,19 +1521,10 @@ "dev": true }, "semver": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", "dev": true - }, - "strip-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", - "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", - "dev": true, - "requires": { - "ansi-regex": "^3.0.0" - } } } }, @@ -1703,9 +1687,9 @@ "dev": true }, "eslint-scope": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", - "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.0.0.tgz", + "integrity": "sha512-oYrhJW7S0bxAFDvWqzvMPRm6pcgcnWc4QnofCAqRTRfQC0JcwenzGglTtsLyIuuWFfkqDG9vz67cnttSd53djw==", "dev": true, "requires": { "esrecurse": "^4.1.0", @@ -1728,14 +1712,22 @@ "dev": true }, "espree": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-5.0.1.tgz", - "integrity": "sha512-qWAZcWh4XE/RwzLJejfcofscgMc9CamR6Tn1+XRXNzrvUSSbiAjGOI/fggztjIi7y9VLPqnICMIPiGyr8JaZ0A==", + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/espree/-/espree-6.1.2.tgz", + "integrity": "sha512-2iUPuuPP+yW1PZaMSDM9eyVf8D5P0Hi8h83YtZ5bPc/zHYjII5khoixIUTMO794NOY8F/ThF1Bo8ncZILarUTA==", "dev": true, "requires": { - "acorn": "^6.0.7", - "acorn-jsx": "^5.0.0", - "eslint-visitor-keys": "^1.0.0" + "acorn": "^7.1.0", + "acorn-jsx": "^5.1.0", + "eslint-visitor-keys": "^1.1.0" + }, + "dependencies": { + "acorn": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.1.0.tgz", + "integrity": "sha512-kL5CuoXA/dgxlBbVrflsflzQ3PAas7RYZB52NOm/6839iVYJgKMJ3cQJD+t2i5+qFa8h3MDpEOJiS64E8JLnSQ==", + "dev": true + } } }, "esprima": { @@ -2194,6 +2186,15 @@ "path-is-absolute": "^1.0.0" } }, + "glob-parent": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.0.tgz", + "integrity": "sha512-qjtRgnIVmOfnKUE3NJAQEdk+lKrxfw8t5ke7SXtfMTHcjsBfOfWXCQfdb30zfDoZQ2IRSIiidmjtbHZPZ++Ihw==", + "dev": true, + "requires": { + "is-glob": "^4.0.1" + } + }, "globals": { "version": "11.12.0", "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", @@ -2506,12 +2507,27 @@ "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY=", "dev": true }, + "is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", + "dev": true + }, "is-fullwidth-code-point": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", "dev": true }, + "is-glob": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz", + "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==", + "dev": true, + "requires": { + "is-extglob": "^2.1.1" + } + }, "is-promise": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz", @@ -4974,9 +4990,9 @@ "dev": true }, "strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.0.1.tgz", + "integrity": "sha512-VTyMAUfdm047mwKl+u79WIdrZxtFtn+nBxHeb844XBQ9uMNTuTHdx2hc5RiAJYqwTj3wc/xe5HLSdJSkJ+WfZw==", "dev": true }, "supports-color": { @@ -5233,6 +5249,12 @@ "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, + "v8-compile-cache": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.0.tgz", + "integrity": "sha512-usZBT3PW+LOjM25wbqIlZwPeJV+3OSz3M1k1Ws8snlW39dZyYL9lOGC5FgPVHfk0jKmjiDV8Z0mIbVQPiwFs7g==", + "dev": true + }, "validate-npm-package-license": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", diff --git a/services/filestore/package.json b/services/filestore/package.json index 47aa4d0d96..fd03757072 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -48,7 +48,7 @@ "babel-eslint": "^10.0.3", "bunyan": "^1.3.5", "chai": "4.2.0", - "eslint": "^5.16.0", + "eslint": "^6.4.0", "eslint-config-prettier": "^6.7.0", "eslint-config-standard": "^14.1.0", "eslint-plugin-chai-expect": "^2.1.0", From eacad771127857339b84cd7e9c7455bd8c3e68f9 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Wed, 18 Dec 2019 10:01:59 +0000 Subject: [PATCH 38/73] Cleanup SafeExec --- services/filestore/app/js/Errors.js | 52 +++++-- services/filestore/app/js/SafeExec.js | 90 ++++++------ .../filestore/test/unit/js/SafeExecTests.js | 134 ++++++++++-------- 3 files changed, 161 insertions(+), 115 deletions(-) diff --git a/services/filestore/app/js/Errors.js b/services/filestore/app/js/Errors.js index f3bc8e37eb..4231571cb3 100644 --- a/services/filestore/app/js/Errors.js +++ b/services/filestore/app/js/Errors.js @@ -1,16 +1,38 @@ -/* eslint-disable - no-proto, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -let Errors -var NotFoundError = function(message) { - const error = new Error(message) - error.name = 'NotFoundError' - error.__proto__ = NotFoundError.prototype - return error -} -NotFoundError.prototype.__proto__ = Error.prototype +const OError = require('@overleaf/o-error') -module.exports = Errors = { NotFoundError } +// Error class for legacy errors so they inherit OError while staying +// backward-compatible (can be instantiated with string as argument instead +// of object) +class BackwardCompatibleError extends OError { + constructor(messageOrOptions) { + let options + if (typeof messageOrOptions === 'string') { + options = { message: messageOrOptions } + } else if (!messageOrOptions) { + options = {} + } else { + options = messageOrOptions + } + super(options) + } +} + +class NotFoundError extends BackwardCompatibleError {} +class ConversionsDisabledError extends BackwardCompatibleError {} + +class FailedCommandError extends OError { + constructor(command, code, stdout, stderr) { + super({ + message: 'command failed with error exit code', + info: { + command, + code + } + }) + this.stdout = stdout + this.stderr = stderr + this.code = code + } +} + +module.exports = { NotFoundError, FailedCommandError, ConversionsDisabledError } diff --git a/services/filestore/app/js/SafeExec.js b/services/filestore/app/js/SafeExec.js index dbc1576a88..5f079fa474 100644 --- a/services/filestore/app/js/SafeExec.js +++ b/services/filestore/app/js/SafeExec.js @@ -1,21 +1,8 @@ -/* eslint-disable - camelcase, - handle-callback-err, - no-return-assign, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ const _ = require('underscore') const logger = require('logger-sharelatex') -const child_process = require('child_process') +const childProcess = require('child_process') const Settings = require('settings-sharelatex') +const { ConversionsDisabledError, FailedCommandError } = require('./Errors') // execute a command in the same way as 'exec' but with a timeout that // kills all child processes @@ -23,36 +10,32 @@ const Settings = require('settings-sharelatex') // we spawn the command with 'detached:true' to make a new process // group, then we can kill everything in that process group. -module.exports = function(command, options, callback) { - if (callback == null) { - callback = function(err, stdout, stderr) {} - } +module.exports = safeExec +module.exports.promises = safeExecPromise + +// options are {timeout: number-of-milliseconds, killSignal: signal-name} +function safeExec(command, options, callback) { if (!Settings.enableConversions) { - const error = new Error('Image conversions are disabled') - return callback(error) + return callback( + new ConversionsDisabledError('image conversions are disabled') + ) } - // options are {timeout: number-of-milliseconds, killSignal: signal-name} - const [cmd, ...args] = Array.from(command) + const [cmd, ...args] = command - const child = child_process.spawn(cmd, args, { detached: true }) + const child = childProcess.spawn(cmd, args, { detached: true }) let stdout = '' let stderr = '' - const cleanup = _.once(function(err) { - if (killTimer != null) { - clearTimeout(killTimer) - } - return callback(err, stdout, stderr) - }) + let killTimer - if (options.timeout != null) { - var killTimer = setTimeout(function() { + if (options.timeout) { + killTimer = setTimeout(function() { try { // use negative process id to kill process group - return process.kill(-child.pid, options.killSignal || 'SIGTERM') + process.kill(-child.pid, options.killSignal || 'SIGTERM') } catch (error) { - return logger.log( + logger.log( { process: child.pid, kill_error: error }, 'error killing process' ) @@ -60,14 +43,41 @@ module.exports = function(command, options, callback) { }, options.timeout) } - child.on('close', function(code, signal) { - const err = code ? new Error(`exit status ${code}`) : signal - return cleanup(err) + const cleanup = _.once(function(err) { + if (killTimer) { + clearTimeout(killTimer) + } + callback(err, stdout, stderr) }) - child.on('error', err => cleanup(err)) + child.on('close', function(code, signal) { + if (code || signal) { + return cleanup( + new FailedCommandError(command, code || signal, stdout, stderr) + ) + } - child.stdout.on('data', chunk => (stdout += chunk)) + cleanup() + }) - return child.stderr.on('data', chunk => (stderr += chunk)) + child.on('error', err => { + cleanup(err) + }) + child.stdout.on('data', chunk => { + stdout += chunk + }) + child.stderr.on('data', chunk => { + stderr += chunk + }) +} + +function safeExecPromise(command, options) { + return new Promise((resolve, reject) => { + safeExec(command, options, (err, stdout, stderr) => { + if (err) { + reject(err) + } + resolve({ stdout, stderr }) + }) + }) } diff --git a/services/filestore/test/unit/js/SafeExecTests.js b/services/filestore/test/unit/js/SafeExecTests.js index 2b629947f5..077964ead7 100644 --- a/services/filestore/test/unit/js/SafeExecTests.js +++ b/services/filestore/test/unit/js/SafeExecTests.js @@ -1,95 +1,109 @@ -/* eslint-disable - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const { assert } = require('chai') -const sinon = require('sinon') const chai = require('chai') const should = chai.should() const { expect } = chai -const modulePath = '../../../app/js/SafeExec.js' +const modulePath = '../../../app/js/SafeExec' const SandboxedModule = require('sandboxed-module') describe('SafeExec', function() { + let settings, options, safeExec + beforeEach(function() { - this.settings = { enableConversions: true } - this.safe_exec = SandboxedModule.require(modulePath, { + settings = { enableConversions: true } + options = { timeout: 10 * 1000, killSignal: 'SIGTERM' } + + safeExec = SandboxedModule.require(modulePath, { requires: { 'logger-sharelatex': { log() {}, err() {} }, - 'settings-sharelatex': this.settings + 'settings-sharelatex': settings } }) - return (this.options = { timeout: 10 * 1000, killSignal: 'SIGTERM' }) }) - return describe('safe_exec', function() { + describe('safeExec', function() { it('should execute a valid command', function(done) { - return this.safe_exec( - ['/bin/echo', 'hello'], - this.options, - (err, stdout, stderr) => { - stdout.should.equal('hello\n') - should.not.exist(err) - return done() - } - ) + safeExec(['/bin/echo', 'hello'], options, (err, stdout, stderr) => { + stdout.should.equal('hello\n') + stderr.should.equal('') + should.not.exist(err) + done() + }) }) it('should error when conversions are disabled', function(done) { - this.settings.enableConversions = false - return this.safe_exec( - ['/bin/echo', 'hello'], - this.options, - (err, stdout, stderr) => { - expect(err).to.exist - return done() - } - ) + settings.enableConversions = false + safeExec(['/bin/echo', 'hello'], options, err => { + expect(err).to.exist + done() + }) }) it('should execute a command with non-zero exit status', function(done) { - return this.safe_exec( - ['/usr/bin/env', 'false'], - this.options, - (err, stdout, stderr) => { - stdout.should.equal('') - stderr.should.equal('') - err.message.should.equal('exit status 1') - return done() - } - ) + safeExec(['/usr/bin/env', 'false'], options, err => { + expect(err).to.exist + expect(err.name).to.equal('FailedCommandError') + expect(err.code).to.equal(1) + expect(err.stdout).to.equal('') + expect(err.stderr).to.equal('') + done() + }) }) it('should handle an invalid command', function(done) { - return this.safe_exec( - ['/bin/foobar'], - this.options, - (err, stdout, stderr) => { - err.code.should.equal('ENOENT') - return done() - } - ) + safeExec(['/bin/foobar'], options, err => { + err.code.should.equal('ENOENT') + done() + }) }) - return it('should handle a command that runs too long', function(done) { - return this.safe_exec( + it('should handle a command that runs too long', function(done) { + safeExec( ['/bin/sleep', '10'], { timeout: 500, killSignal: 'SIGTERM' }, - (err, stdout, stderr) => { - err.should.equal('SIGTERM') - return done() + err => { + expect(err).to.exist + expect(err.name).to.equal('FailedCommandError') + expect(err.code).to.equal('SIGTERM') + done() } ) }) }) + + describe('as a promise', function() { + beforeEach(function() { + safeExec = safeExec.promises + }) + + it('should execute a valid command', async function() { + const { stdout, stderr } = await safeExec(['/bin/echo', 'hello'], options) + + stdout.should.equal('hello\n') + stderr.should.equal('') + }) + + it('should throw a ConversionsDisabledError when appropriate', async function() { + settings.enableConversions = false + try { + await safeExec(['/bin/echo', 'hello'], options) + } catch (err) { + expect(err.name).to.equal('ConversionsDisabledError') + return + } + expect('method did not throw an error').not.to.exist + }) + + it('should throw a FailedCommandError when appropriate', async function() { + try { + await safeExec(['/usr/bin/env', 'false'], options) + } catch (err) { + expect(err.name).to.equal('FailedCommandError') + expect(err.code).to.equal(1) + return + } + expect('method did not throw an error').not.to.exist + }) + }) }) From 27aaff7843c6dd39e1623b00402939effa3f7afb Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Wed, 18 Dec 2019 15:40:30 +0000 Subject: [PATCH 39/73] Decaf cleanup for FileHandler and LocalFileWriter Simplified code and tests where possible --- services/filestore/app/js/Errors.js | 12 +- services/filestore/app/js/FileHandler.js | 417 +++++++-------- services/filestore/app/js/LocalFileWriter.js | 120 ++--- services/filestore/npm-shrinkwrap.json | 6 + services/filestore/package.json | 3 +- .../test/unit/js/FileHandlerTests.js | 476 +++++++----------- .../test/unit/js/LocalFileWriterTests.js | 139 ++--- 7 files changed, 474 insertions(+), 699 deletions(-) diff --git a/services/filestore/app/js/Errors.js b/services/filestore/app/js/Errors.js index 4231571cb3..57dbdbe522 100644 --- a/services/filestore/app/js/Errors.js +++ b/services/filestore/app/js/Errors.js @@ -18,7 +18,10 @@ class BackwardCompatibleError extends OError { } class NotFoundError extends BackwardCompatibleError {} +class WriteError extends BackwardCompatibleError {} +class ReadError extends BackwardCompatibleError {} class ConversionsDisabledError extends BackwardCompatibleError {} +class ConversionError extends BackwardCompatibleError {} class FailedCommandError extends OError { constructor(command, code, stdout, stderr) { @@ -35,4 +38,11 @@ class FailedCommandError extends OError { } } -module.exports = { NotFoundError, FailedCommandError, ConversionsDisabledError } +module.exports = { + NotFoundError, + FailedCommandError, + ConversionsDisabledError, + WriteError, + ReadError, + ConversionError +} diff --git a/services/filestore/app/js/FileHandler.js b/services/filestore/app/js/FileHandler.js index e63c813167..3e102b316b 100644 --- a/services/filestore/app/js/FileHandler.js +++ b/services/filestore/app/js/FileHandler.js @@ -1,18 +1,5 @@ -/* eslint-disable - camelcase, - no-self-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -let FileHandler -const settings = require('settings-sharelatex') +const { promisify } = require('util') +const fs = require('fs') const PersistorManager = require('./PersistorManager') const LocalFileWriter = require('./LocalFileWriter') const logger = require('logger-sharelatex') @@ -20,216 +7,196 @@ const FileConverter = require('./FileConverter') const KeyBuilder = require('./KeyBuilder') const async = require('async') const ImageOptimiser = require('./ImageOptimiser') -const Errors = require('./Errors') +const { WriteError, ReadError, ConversionError } = require('./Errors') -module.exports = FileHandler = { - insertFile(bucket, key, stream, callback) { - const convertedKey = KeyBuilder.getConvertedFolderKey(key) - return PersistorManager.deleteDirectory(bucket, convertedKey, function( - error - ) { - if (error != null) { - return callback(error) - } - return PersistorManager.sendStream(bucket, key, stream, callback) - }) - }, - - deleteFile(bucket, key, callback) { - const convertedKey = KeyBuilder.getConvertedFolderKey(key) - return async.parallel( - [ - done => PersistorManager.deleteFile(bucket, key, done), - done => PersistorManager.deleteDirectory(bucket, convertedKey, done) - ], - callback - ) - }, - - getFile(bucket, key, opts, callback) { - // In this call, opts can contain credentials - if (opts == null) { - opts = {} - } - logger.log({ bucket, key, opts: this._scrubSecrets(opts) }, 'getting file') - if (opts.format == null && opts.style == null) { - return this._getStandardFile(bucket, key, opts, callback) - } else { - return this._getConvertedFile(bucket, key, opts, callback) - } - }, - - getFileSize(bucket, key, callback) { - return PersistorManager.getFileSize(bucket, key, callback) - }, - - _getStandardFile(bucket, key, opts, callback) { - return PersistorManager.getFileStream(bucket, key, opts, function( - err, - fileStream - ) { - if (err != null && !(err instanceof Errors.NotFoundError)) { - logger.err( - { bucket, key, opts: FileHandler._scrubSecrets(opts) }, - 'error getting fileStream' - ) - } - return callback(err, fileStream) - }) - }, - - _getConvertedFile(bucket, key, opts, callback) { - const convertedKey = KeyBuilder.addCachingToKey(key, opts) - return PersistorManager.checkIfFileExists( - bucket, - convertedKey, - (err, exists) => { - if (err != null) { - return callback(err) - } - if (exists) { - return PersistorManager.getFileStream( - bucket, - convertedKey, - opts, - callback - ) - } else { - return this._getConvertedFileAndCache( - bucket, - key, - convertedKey, - opts, - callback - ) - } - } - ) - }, - - _getConvertedFileAndCache(bucket, key, convertedKey, opts, callback) { - let convertedFsPath = '' - const originalFsPath = '' - return async.series( - [ - cb => { - return this._convertFile(bucket, key, opts, function( - err, - fileSystemPath, - originalFsPath - ) { - convertedFsPath = fileSystemPath - originalFsPath = originalFsPath - return cb(err) - }) - }, - cb => ImageOptimiser.compressPng(convertedFsPath, cb), - cb => - PersistorManager.sendFile(bucket, convertedKey, convertedFsPath, cb) - ], - function(err) { - if (err != null) { - LocalFileWriter.deleteFile(convertedFsPath, function() {}) - LocalFileWriter.deleteFile(originalFsPath, function() {}) - return callback(err) - } - // Send back the converted file from the local copy to avoid problems - // with the file not being present in S3 yet. As described in the - // documentation below, we have already made a 'HEAD' request in - // checkIfFileExists so we only have "eventual consistency" if we try - // to stream it from S3 here. This was a cause of many 403 errors. - // - // "Amazon S3 provides read-after-write consistency for PUTS of new - // objects in your S3 bucket in all regions with one caveat. The - // caveat is that if you make a HEAD or GET request to the key name - // (to find if the object exists) before creating the object, Amazon - // S3 provides eventual consistency for read-after-write."" - // https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel - return LocalFileWriter.getStream(convertedFsPath, function( - err, - readStream - ) { - if (err != null) { - return callback(err) - } - readStream.on('end', function() { - logger.log({ convertedFsPath }, 'deleting temporary file') - return LocalFileWriter.deleteFile(convertedFsPath, function() {}) - }) - return callback(null, readStream) - }) - } - ) - }, - - _convertFile(bucket, originalKey, opts, callback) { - return this._writeS3FileToDisk(bucket, originalKey, opts, function( - err, - originalFsPath - ) { - if (err != null) { - return callback(err) - } - const done = function(err, destPath) { - if (err != null) { - logger.err( - { err, bucket, originalKey, opts: FileHandler._scrubSecrets(opts) }, - 'error converting file' - ) - return callback(err) - } - LocalFileWriter.deleteFile(originalFsPath, function() {}) - return callback(err, destPath, originalFsPath) - } - - logger.log({ opts }, 'converting file depending on opts') - - if (opts.format != null) { - return FileConverter.convert(originalFsPath, opts.format, done) - } else if (opts.style === 'thumbnail') { - return FileConverter.thumbnail(originalFsPath, done) - } else if (opts.style === 'preview') { - return FileConverter.preview(originalFsPath, done) - } else { - return callback( - new Error( - `should have specified opts to convert file with ${JSON.stringify( - opts - )}` - ) - ) - } - }) - }, - - _writeS3FileToDisk(bucket, key, opts, callback) { - return PersistorManager.getFileStream(bucket, key, opts, function( - err, - fileStream - ) { - if (err != null) { - return callback(err) - } - return LocalFileWriter.writeStream(fileStream, key, callback) - }) - }, - - getDirectorySize(bucket, project_id, callback) { - logger.log({ bucket, project_id }, 'getting project size') - return PersistorManager.directorySize(bucket, project_id, function( - err, - size - ) { - if (err != null) { - logger.err({ bucket, project_id }, 'error getting size') - } - return callback(err, size) - }) - }, - - _scrubSecrets(opts) { - const safe = Object.assign({}, opts) - delete safe.credentials - return safe +module.exports = { + insertFile, + deleteFile, + getFile, + getFileSize, + getDirectorySize, + promises: { + getFile: promisify(getFile), + insertFile: promisify(insertFile), + deleteFile: promisify(deleteFile), + getFileSize: promisify(getFileSize), + getDirectorySize: promisify(getDirectorySize) } } + +function insertFile(bucket, key, stream, callback) { + const convertedKey = KeyBuilder.getConvertedFolderKey(key) + PersistorManager.deleteDirectory(bucket, convertedKey, function(error) { + if (error) { + return callback(new WriteError('error inserting file').withCause(error)) + } + PersistorManager.sendStream(bucket, key, stream, callback) + }) +} + +function deleteFile(bucket, key, callback) { + const convertedKey = KeyBuilder.getConvertedFolderKey(key) + async.parallel( + [ + done => PersistorManager.deleteFile(bucket, key, done), + done => PersistorManager.deleteDirectory(bucket, convertedKey, done) + ], + callback + ) +} + +function getFile(bucket, key, opts, callback) { + // In this call, opts can contain credentials + if (!opts) { + opts = {} + } + logger.log({ bucket, key, opts: _scrubSecrets(opts) }, 'getting file') + if (!opts.format && !opts.style) { + _getStandardFile(bucket, key, opts, callback) + } else { + _getConvertedFile(bucket, key, opts, callback) + } +} + +function getFileSize(bucket, key, callback) { + PersistorManager.getFileSize(bucket, key, callback) +} + +function getDirectorySize(bucket, projectId, callback) { + logger.log({ bucket, project_id: projectId }, 'getting project size') + PersistorManager.directorySize(bucket, projectId, function(err, size) { + if (err) { + logger.err({ bucket, project_id: projectId }, 'error getting size') + err = new ReadError('error getting project size').withCause(err) + } + return callback(err, size) + }) +} + +function _getStandardFile(bucket, key, opts, callback) { + PersistorManager.getFileStream(bucket, key, opts, function(err, fileStream) { + if (err && err.name !== 'NotFoundError') { + logger.err( + { bucket, key, opts: _scrubSecrets(opts) }, + 'error getting fileStream' + ) + } + callback(err, fileStream) + }) +} + +function _getConvertedFile(bucket, key, opts, callback) { + const convertedKey = KeyBuilder.addCachingToKey(key, opts) + PersistorManager.checkIfFileExists(bucket, convertedKey, (err, exists) => { + if (err) { + return callback(err) + } + + if (exists) { + PersistorManager.getFileStream(bucket, convertedKey, opts, callback) + } else { + _getConvertedFileAndCache(bucket, key, convertedKey, opts, callback) + } + }) +} + +function _getConvertedFileAndCache(bucket, key, convertedKey, opts, callback) { + let convertedFsPath + + async.series( + [ + cb => { + _convertFile(bucket, key, opts, function(err, fileSystemPath) { + convertedFsPath = fileSystemPath + cb(err) + }) + }, + cb => ImageOptimiser.compressPng(convertedFsPath, cb), + cb => PersistorManager.sendFile(bucket, convertedKey, convertedFsPath, cb) + ], + function(err) { + if (err) { + LocalFileWriter.deleteFile(convertedFsPath, function() {}) + return callback( + new ConversionError('failed to convert file').withCause(err) + ) + } + // Send back the converted file from the local copy to avoid problems + // with the file not being present in S3 yet. As described in the + // documentation below, we have already made a 'HEAD' request in + // checkIfFileExists so we only have "eventual consistency" if we try + // to stream it from S3 here. This was a cause of many 403 errors. + // + // "Amazon S3 provides read-after-write consistency for PUTS of new + // objects in your S3 bucket in all regions with one caveat. The + // caveat is that if you make a HEAD or GET request to the key name + // (to find if the object exists) before creating the object, Amazon + // S3 provides eventual consistency for read-after-write."" + // https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel + const readStream = fs.createReadStream(convertedFsPath) + readStream.on('end', function() { + LocalFileWriter.deleteFile(convertedFsPath, function() {}) + }) + callback(null, readStream) + } + ) +} + +function _convertFile(bucket, originalKey, opts, callback) { + _writeFileToDisk(bucket, originalKey, opts, function(err, originalFsPath) { + if (err) { + return callback( + new ConversionError('unable to write file to disk').withCause(err) + ) + } + + const done = function(err, destPath) { + if (err) { + logger.err( + { err, bucket, originalKey, opts: _scrubSecrets(opts) }, + 'error converting file' + ) + return callback( + new ConversionError('error converting file').withCause(err) + ) + } + LocalFileWriter.deleteFile(originalFsPath, function() {}) + callback(err, destPath) + } + + logger.log({ opts }, 'converting file depending on opts') + + if (opts.format) { + FileConverter.convert(originalFsPath, opts.format, done) + } else if (opts.style === 'thumbnail') { + FileConverter.thumbnail(originalFsPath, done) + } else if (opts.style === 'preview') { + FileConverter.preview(originalFsPath, done) + } else { + callback( + new ConversionError( + `should have specified opts to convert file with ${JSON.stringify( + opts + )}` + ) + ) + } + }) +} + +function _writeFileToDisk(bucket, key, opts, callback) { + PersistorManager.getFileStream(bucket, key, opts, function(err, fileStream) { + if (err) { + return callback( + new ReadError('unable to get read stream for file').withCause(err) + ) + } + LocalFileWriter.writeStream(fileStream, key, callback) + }) +} + +function _scrubSecrets(opts) { + const safe = Object.assign({}, opts) + delete safe.credentials + return safe +} diff --git a/services/filestore/app/js/LocalFileWriter.js b/services/filestore/app/js/LocalFileWriter.js index 8a541a35e9..44f3f9433a 100644 --- a/services/filestore/app/js/LocalFileWriter.js +++ b/services/filestore/app/js/LocalFileWriter.js @@ -1,91 +1,57 @@ -/* eslint-disable - handle-callback-err, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ const fs = require('fs') const uuid = require('node-uuid') const path = require('path') -const _ = require('underscore') +const Stream = require('stream') +const { callbackify, promisify } = require('util') const logger = require('logger-sharelatex') const metrics = require('metrics-sharelatex') const Settings = require('settings-sharelatex') -const Errors = require('./Errors') +const { WriteError } = require('./Errors') module.exports = { - writeStream(stream, key, callback) { - const timer = new metrics.Timer('writingFile') - callback = _.once(callback) - const fsPath = this._getPath(key) - logger.log({ fsPath }, 'writing file locally') - const writeStream = fs.createWriteStream(fsPath) - writeStream.on('finish', function() { - timer.done() - logger.log({ fsPath }, 'finished writing file locally') - return callback(null, fsPath) - }) - writeStream.on('error', function(err) { - logger.err( - { err, fsPath }, - 'problem writing file locally, with write stream' - ) - return callback(err) - }) - stream.on('error', function(err) { - logger.log( - { err, fsPath }, - 'problem writing file locally, with read stream' - ) - return callback(err) - }) - return stream.pipe(writeStream) + promises: { + writeStream, + deleteFile }, + writeStream: callbackify(writeStream), + deleteFile: callbackify(deleteFile) +} - getStream(fsPath, _callback) { - if (_callback == null) { - _callback = function(err, res) {} - } - const callback = _.once(_callback) - const timer = new metrics.Timer('readingFile') - logger.log({ fsPath }, 'reading file locally') - const readStream = fs.createReadStream(fsPath) - readStream.on('end', function() { - timer.done() - return logger.log({ fsPath }, 'finished reading file locally') - }) - readStream.on('error', function(err) { - logger.err( - { err, fsPath }, - 'problem reading file locally, with read stream' - ) - if (err.code === 'ENOENT') { - return callback(new Errors.NotFoundError(err.message), null) - } else { - return callback(err) - } - }) - return callback(null, readStream) - }, +const pipeline = promisify(Stream.pipeline) - deleteFile(fsPath, callback) { - if (fsPath == null || fsPath === '') { - return callback() - } - logger.log({ fsPath }, 'removing local temp file') - return fs.unlink(fsPath, callback) - }, +async function writeStream(stream, key) { + const timer = new metrics.Timer('writingFile') + const fsPath = _getPath(key) - _getPath(key) { - if (key == null) { - key = uuid.v1() - } - key = key.replace(/\//g, '-') - return path.join(Settings.path.uploadFolder, key) + logger.log({ fsPath }, 'writing file locally') + + const writeStream = fs.createWriteStream(fsPath) + try { + await pipeline(stream, writeStream) + timer.done() + logger.log({ fsPath }, 'finished writing file locally') + return fsPath + } catch (err) { + logger.err({ err, fsPath }, 'problem writing file locally') + throw new WriteError({ + message: 'problem writing file locally', + info: { err, fsPath } + }).withCause(err) } } + +async function deleteFile(fsPath) { + if (!fsPath) { + return + } + logger.log({ fsPath }, 'removing local temp file') + await promisify(fs.unlink)(fsPath) +} + +function _getPath(key) { + if (key == null) { + key = uuid.v1() + } + key = key.replace(/\//g, '-') + return path.join(Settings.path.uploadFolder, key) +} diff --git a/services/filestore/npm-shrinkwrap.json b/services/filestore/npm-shrinkwrap.json index ef0f78fc15..44f5ec3263 100644 --- a/services/filestore/npm-shrinkwrap.json +++ b/services/filestore/npm-shrinkwrap.json @@ -4763,6 +4763,12 @@ "type-detect": "^4.0.8" } }, + "sinon-chai": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/sinon-chai/-/sinon-chai-3.3.0.tgz", + "integrity": "sha512-r2JhDY7gbbmh5z3Q62pNbrjxZdOAjpsqW/8yxAZRSqLZqowmfGZPGUZPFf3UX36NLis0cv8VEM5IJh9HgkSOAA==", + "dev": true + }, "slice-ansi": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz", diff --git a/services/filestore/package.json b/services/filestore/package.json index fd03757072..d39d1027be 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -62,6 +62,7 @@ "prettier-eslint": "^9.0.1", "prettier-eslint-cli": "^5.0.0", "sandboxed-module": "2.0.3", - "sinon": "7.1.1" + "sinon": "7.1.1", + "sinon-chai": "^3.3.0" } } diff --git a/services/filestore/test/unit/js/FileHandlerTests.js b/services/filestore/test/unit/js/FileHandlerTests.js index e641ffdd16..671e5c41ea 100644 --- a/services/filestore/test/unit/js/FileHandlerTests.js +++ b/services/filestore/test/unit/js/FileHandlerTests.js @@ -1,367 +1,233 @@ -/* eslint-disable - handle-callback-err, - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const { assert } = require('chai') const sinon = require('sinon') const chai = require('chai') -const should = chai.should() const { expect } = chai const modulePath = '../../../app/js/FileHandler.js' const SandboxedModule = require('sandboxed-module') describe('FileHandler', function() { - beforeEach(function() { - this.settings = { - s3: { - buckets: { - user_files: 'user_files' - } + let PersistorManager, + LocalFileWriter, + FileConverter, + KeyBuilder, + ImageOptimiser, + FileHandler, + fs + const settings = { + s3: { + buckets: { + user_files: 'user_files' } } - this.PersistorManager = { - getFileStream: sinon.stub(), - checkIfFileExists: sinon.stub(), - deleteFile: sinon.stub(), - deleteDirectory: sinon.stub(), - sendStream: sinon.stub(), - insertFile: sinon.stub(), - directorySize: sinon.stub() + } + + const bucket = 'my_bucket' + const key = 'key/here' + const convertedFolderKey = 'convertedFolder' + const sourceStream = 'sourceStream' + const convertedKey = 'convertedKey' + const readStream = { + stream: 'readStream', + on: sinon.stub() + } + + beforeEach(function() { + PersistorManager = { + getFileStream: sinon.stub().yields(null, sourceStream), + checkIfFileExists: sinon.stub().yields(), + deleteFile: sinon.stub().yields(), + deleteDirectory: sinon.stub().yields(), + sendStream: sinon.stub().yields(), + insertFile: sinon.stub().yields(), + sendFile: sinon.stub().yields(), + directorySize: sinon.stub().yields() } - this.LocalFileWriter = { - writeStream: sinon.stub(), - getStream: sinon.stub(), - deleteFile: sinon.stub() + LocalFileWriter = { + writeStream: sinon.stub().yields(), + deleteFile: sinon.stub().yields() } - this.FileConverter = { - convert: sinon.stub(), - thumbnail: sinon.stub(), - preview: sinon.stub() + FileConverter = { + convert: sinon.stub().yields(), + thumbnail: sinon.stub().yields(), + preview: sinon.stub().yields() } - this.keyBuilder = { - addCachingToKey: sinon.stub(), - getConvertedFolderKey: sinon.stub() + KeyBuilder = { + addCachingToKey: sinon.stub().returns(convertedKey), + getConvertedFolderKey: sinon.stub().returns(convertedFolderKey) } - this.ImageOptimiser = { compressPng: sinon.stub() } - this.handler = SandboxedModule.require(modulePath, { + ImageOptimiser = { compressPng: sinon.stub().yields() } + fs = { + createReadStream: sinon.stub().returns(readStream) + } + + FileHandler = SandboxedModule.require(modulePath, { requires: { - 'settings-sharelatex': this.settings, - './PersistorManager': this.PersistorManager, - './LocalFileWriter': this.LocalFileWriter, - './FileConverter': this.FileConverter, - './KeyBuilder': this.keyBuilder, - './ImageOptimiser': this.ImageOptimiser, + 'settings-sharelatex': settings, + './PersistorManager': PersistorManager, + './LocalFileWriter': LocalFileWriter, + './FileConverter': FileConverter, + './KeyBuilder': KeyBuilder, + './ImageOptimiser': ImageOptimiser, + fs: fs, 'logger-sharelatex': { log() {}, err() {} } - } + }, + globals: { console } }) - this.bucket = 'my_bucket' - this.key = 'key/here' - this.stubbedPath = '/var/somewhere/path' - this.format = 'png' - return (this.formattedStubbedPath = `${this.stubbedPath}.${this.format}`) }) describe('insertFile', function() { - beforeEach(function() { - this.stream = {} - this.PersistorManager.deleteDirectory.callsArgWith(2) - return this.PersistorManager.sendStream.callsArgWith(3) - }) + const stream = 'stream' it('should send file to the filestore', function(done) { - return this.handler.insertFile(this.bucket, this.key, this.stream, () => { - this.PersistorManager.sendStream - .calledWith(this.bucket, this.key, this.stream) - .should.equal(true) - return done() + FileHandler.insertFile(bucket, key, stream, err => { + expect(err).not.to.exist + expect(PersistorManager.sendStream).to.have.been.calledWith( + bucket, + key, + stream + ) + done() }) }) - return it('should delete the convetedKey folder', function(done) { - this.keyBuilder.getConvertedFolderKey.returns(this.stubbedConvetedKey) - return this.handler.insertFile(this.bucket, this.key, this.stream, () => { - this.PersistorManager.deleteDirectory - .calledWith(this.bucket, this.stubbedConvetedKey) - .should.equal(true) - return done() + it('should delete the convertedKey folder', function(done) { + FileHandler.insertFile(bucket, key, stream, err => { + expect(err).not.to.exist + expect(PersistorManager.deleteDirectory).to.have.been.calledWith( + bucket, + convertedFolderKey + ) + done() }) }) }) describe('deleteFile', function() { - beforeEach(function() { - this.keyBuilder.getConvertedFolderKey.returns(this.stubbedConvetedKey) - this.PersistorManager.deleteFile.callsArgWith(2) - return this.PersistorManager.deleteDirectory.callsArgWith(2) - }) - it('should tell the filestore manager to delete the file', function(done) { - return this.handler.deleteFile(this.bucket, this.key, () => { - this.PersistorManager.deleteFile - .calledWith(this.bucket, this.key) - .should.equal(true) - return done() + FileHandler.deleteFile(bucket, key, err => { + expect(err).not.to.exist + expect(PersistorManager.deleteFile).to.have.been.calledWith(bucket, key) + done() }) }) - return it('should tell the filestore manager to delete the cached foler', function(done) { - return this.handler.deleteFile(this.bucket, this.key, () => { - this.PersistorManager.deleteDirectory - .calledWith(this.bucket, this.stubbedConvetedKey) - .should.equal(true) - return done() + it('should tell the filestore manager to delete the cached folder', function(done) { + FileHandler.deleteFile(bucket, key, err => { + expect(err).not.to.exist + expect(PersistorManager.deleteDirectory).to.have.been.calledWith( + bucket, + convertedFolderKey + ) + done() }) }) }) describe('getFile', function() { - beforeEach(function() { - this.handler._getStandardFile = sinon.stub().callsArgWith(3) - return (this.handler._getConvertedFile = sinon.stub().callsArgWith(3)) - }) - - it('should call _getStandardFile if no format or style are defined', function(done) { - return this.handler.getFile(this.bucket, this.key, null, () => { - this.handler._getStandardFile.called.should.equal(true) - this.handler._getConvertedFile.called.should.equal(false) - return done() + it('should return the source stream no format or style are defined', function(done) { + FileHandler.getFile(bucket, key, null, (err, stream) => { + expect(err).not.to.exist + expect(stream).to.equal(sourceStream) + done() }) }) - it('should pass options to _getStandardFile', function(done) { + it('should pass options through to PersistorManager', function(done) { const options = { start: 0, end: 8 } - return this.handler.getFile(this.bucket, this.key, options, () => { - expect(this.handler._getStandardFile.lastCall.args[2].start).to.equal(0) - expect(this.handler._getStandardFile.lastCall.args[2].end).to.equal(8) - return done() + FileHandler.getFile(bucket, key, options, err => { + expect(err).not.to.exist + expect(PersistorManager.getFileStream).to.have.been.calledWith( + bucket, + key, + options + ) + done() }) }) - return it('should call _getConvertedFile if a format is defined', function(done) { - return this.handler.getFile( - this.bucket, - this.key, - { format: 'png' }, - () => { - this.handler._getStandardFile.called.should.equal(false) - this.handler._getConvertedFile.called.should.equal(true) - return done() - } - ) - }) - }) + describe('when a format is defined', function() { + let result - describe('_getStandardFile', function() { - beforeEach(function() { - this.fileStream = { on() {} } - return this.PersistorManager.getFileStream.callsArgWith( - 3, - 'err', - this.fileStream - ) - }) + describe('when the file is not cached', function() { + beforeEach(function(done) { + FileHandler.getFile(bucket, key, { format: 'png' }, (err, stream) => { + result = { err, stream } + done() + }) + }) - it('should get the stream', function(done) { - return this.handler.getFile(this.bucket, this.key, null, () => { - this.PersistorManager.getFileStream - .calledWith(this.bucket, this.key) - .should.equal(true) - return done() + it('should convert the file', function() { + expect(FileConverter.convert).to.have.been.called + expect(ImageOptimiser.compressPng).to.have.been.called + }) + + it('should return the the converted stream', function() { + expect(result.err).not.to.exist + expect(result.stream).to.equal(readStream) + expect(PersistorManager.getFileStream).to.have.been.calledWith( + bucket, + key + ) + }) + }) + + describe('when the file is cached', function() { + beforeEach(function(done) { + PersistorManager.checkIfFileExists = sinon.stub().yields(null, true) + FileHandler.getFile(bucket, key, { format: 'png' }, (err, stream) => { + result = { err, stream } + done() + }) + }) + + it('should not convert the file', function() { + expect(FileConverter.convert).not.to.have.been.called + expect(ImageOptimiser.compressPng).not.to.have.been.called + }) + + it('should return the cached stream', function() { + expect(result.err).not.to.exist + expect(result.stream).to.equal(sourceStream) + expect(PersistorManager.getFileStream).to.have.been.calledWith( + bucket, + convertedKey + ) + }) }) }) - it('should return the stream and error', function(done) { - return this.handler.getFile( - this.bucket, - this.key, - null, - (err, stream) => { - err.should.equal('err') - stream.should.equal(this.fileStream) - return done() - } - ) - }) - - return it('should pass options to PersistorManager', function(done) { - return this.handler.getFile( - this.bucket, - this.key, - { start: 0, end: 8 }, - () => { - expect( - this.PersistorManager.getFileStream.lastCall.args[2].start - ).to.equal(0) - expect( - this.PersistorManager.getFileStream.lastCall.args[2].end - ).to.equal(8) - return done() - } - ) - }) - }) - - describe('_getConvertedFile', function() { - it('should getFileStream if it does exists', function(done) { - this.PersistorManager.checkIfFileExists.callsArgWith(2, null, true) - this.PersistorManager.getFileStream.callsArgWith(3) - return this.handler._getConvertedFile(this.bucket, this.key, {}, () => { - this.PersistorManager.getFileStream - .calledWith(this.bucket) - .should.equal(true) - return done() + describe('when a style is defined', function() { + it('generates a thumbnail when requested', function(done) { + FileHandler.getFile(bucket, key, { style: 'thumbnail' }, err => { + expect(err).not.to.exist + expect(FileConverter.thumbnail).to.have.been.called + expect(FileConverter.preview).not.to.have.been.called + done() + }) }) - }) - return it('should call _getConvertedFileAndCache if it does exists', function(done) { - this.PersistorManager.checkIfFileExists.callsArgWith(2, null, false) - this.handler._getConvertedFileAndCache = sinon.stub().callsArgWith(4) - return this.handler._getConvertedFile(this.bucket, this.key, {}, () => { - this.handler._getConvertedFileAndCache - .calledWith(this.bucket, this.key) - .should.equal(true) - return done() + it('generates a preview when requested', function(done) { + FileHandler.getFile(bucket, key, { style: 'preview' }, err => { + expect(err).not.to.exist + expect(FileConverter.thumbnail).not.to.have.been.called + expect(FileConverter.preview).to.have.been.called + done() + }) }) }) }) - describe('_getConvertedFileAndCache', () => - it('should _convertFile ', function(done) { - this.stubbedStream = { something: 'here' } - this.localStream = { - on() {} - } - this.PersistorManager.sendFile = sinon.stub().callsArgWith(3) - this.LocalFileWriter.getStream = sinon - .stub() - .callsArgWith(1, null, this.localStream) - this.convetedKey = this.key + 'converted' - this.handler._convertFile = sinon - .stub() - .callsArgWith(3, null, this.stubbedPath) - this.ImageOptimiser.compressPng = sinon.stub().callsArgWith(1) - return this.handler._getConvertedFileAndCache( - this.bucket, - this.key, - this.convetedKey, - {}, - (err, fsStream) => { - this.handler._convertFile.called.should.equal(true) - this.PersistorManager.sendFile - .calledWith(this.bucket, this.convetedKey, this.stubbedPath) - .should.equal(true) - this.ImageOptimiser.compressPng - .calledWith(this.stubbedPath) - .should.equal(true) - this.LocalFileWriter.getStream - .calledWith(this.stubbedPath) - .should.equal(true) - fsStream.should.equal(this.localStream) - return done() - } - ) - })) - - describe('_convertFile', function() { - beforeEach(function() { - this.FileConverter.convert.callsArgWith( - 2, - null, - this.formattedStubbedPath - ) - this.FileConverter.thumbnail.callsArgWith( - 1, - null, - this.formattedStubbedPath - ) - this.FileConverter.preview.callsArgWith( - 1, - null, - this.formattedStubbedPath - ) - this.handler._writeS3FileToDisk = sinon - .stub() - .callsArgWith(3, null, this.stubbedPath) - return this.LocalFileWriter.deleteFile.callsArgWith(1) - }) - - it('should call thumbnail on the writer path if style was thumbnail was specified', function(done) { - return this.handler._convertFile( - this.bucket, - this.key, - { style: 'thumbnail' }, - (err, path) => { - path.should.equal(this.formattedStubbedPath) - this.FileConverter.thumbnail - .calledWith(this.stubbedPath) - .should.equal(true) - this.LocalFileWriter.deleteFile - .calledWith(this.stubbedPath) - .should.equal(true) - return done() - } - ) - }) - - it('should call preview on the writer path if style was preview was specified', function(done) { - return this.handler._convertFile( - this.bucket, - this.key, - { style: 'preview' }, - (err, path) => { - path.should.equal(this.formattedStubbedPath) - this.FileConverter.preview - .calledWith(this.stubbedPath) - .should.equal(true) - this.LocalFileWriter.deleteFile - .calledWith(this.stubbedPath) - .should.equal(true) - return done() - } - ) - }) - - return it('should call convert on the writer path if a format was specified', function(done) { - return this.handler._convertFile( - this.bucket, - this.key, - { format: this.format }, - (err, path) => { - path.should.equal(this.formattedStubbedPath) - this.FileConverter.convert - .calledWith(this.stubbedPath, this.format) - .should.equal(true) - this.LocalFileWriter.deleteFile - .calledWith(this.stubbedPath) - .should.equal(true) - return done() - } - ) - }) - }) - - return describe('getDirectorySize', function() { - beforeEach(function() { - return this.PersistorManager.directorySize.callsArgWith(2) - }) - - return it('should call the filestore manager to get directory size', function(done) { - return this.handler.getDirectorySize(this.bucket, this.key, () => { - this.PersistorManager.directorySize - .calledWith(this.bucket, this.key) - .should.equal(true) - return done() + describe('getDirectorySize', function() { + it('should call the filestore manager to get directory size', function(done) { + FileHandler.getDirectorySize(bucket, key, err => { + expect(err).not.to.exist + expect(PersistorManager.directorySize).to.have.been.calledWith( + bucket, + key + ) + done() }) }) }) diff --git a/services/filestore/test/unit/js/LocalFileWriterTests.js b/services/filestore/test/unit/js/LocalFileWriterTests.js index 04cc2fb049..5d7008a91f 100644 --- a/services/filestore/test/unit/js/LocalFileWriterTests.js +++ b/services/filestore/test/unit/js/LocalFileWriterTests.js @@ -1,120 +1,79 @@ -/* eslint-disable - handle-callback-err, - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ - -const { assert } = require('chai') const sinon = require('sinon') const chai = require('chai') -const should = chai.should() const { expect } = chai const modulePath = '../../../app/js/LocalFileWriter.js' const SandboxedModule = require('sandboxed-module') +chai.use(require('sinon-chai')) describe('LocalFileWriter', function() { + const writeStream = 'writeStream' + const readStream = 'readStream' + const settings = { path: { uploadFolder: '/uploads' } } + const fsPath = '/uploads/wombat' + const filename = 'wombat' + let stream, fs, LocalFileWriter + beforeEach(function() { - this.writeStream = { - on(type, cb) { - if (type === 'finish') { - return cb() - } - } + fs = { + createWriteStream: sinon.stub().returns(writeStream), + unlink: sinon.stub().yields() } - this.readStream = { on() {} } - this.fs = { - createWriteStream: sinon.stub().returns(this.writeStream), - createReadStream: sinon.stub().returns(this.readStream), - unlink: sinon.stub() + stream = { + pipeline: sinon.stub().yields() } - this.settings = { - path: { - uploadFolder: 'somewhere' - } - } - this.writer = SandboxedModule.require(modulePath, { + + LocalFileWriter = SandboxedModule.require(modulePath, { requires: { - fs: this.fs, + fs, + stream, 'logger-sharelatex': { log() {}, err() {} }, - 'settings-sharelatex': this.settings, + 'settings-sharelatex': settings, 'metrics-sharelatex': { inc: sinon.stub(), Timer: sinon.stub().returns({ done: sinon.stub() }) } } }) - - return (this.stubbedFsPath = 'something/uploads/eio2k1j3') }) - describe('writeStrem', function() { - beforeEach(function() { - return (this.writer._getPath = sinon.stub().returns(this.stubbedFsPath)) - }) - - it('write the stream to ./uploads', function(done) { - const stream = { - pipe: dest => { - dest.should.equal(this.writeStream) - return done() - }, - on() {} - } - return this.writer.writeStream(stream, null, () => {}) - }) - - return it('should send the path in the callback', function(done) { - const stream = { - pipe: dest => {}, - on(type, cb) { - if (type === 'end') { - return cb() - } - } - } - return this.writer.writeStream(stream, null, (err, fsPath) => { - fsPath.should.equal(this.stubbedFsPath) - return done() + describe('writeStream', function() { + it('writes the stream to the upload folder', function(done) { + LocalFileWriter.writeStream(readStream, filename, (err, path) => { + expect(err).not.to.exist + expect(fs.createWriteStream).to.have.been.calledWith(fsPath) + expect(stream.pipeline).to.have.been.calledWith(readStream, writeStream) + expect(path).to.equal(fsPath) + done() }) }) }) - describe('getStream', function() { - it('should read the stream from the file ', function(done) { - return this.writer.getStream(this.stubbedFsPath, (err, stream) => { - this.fs.createReadStream - .calledWith(this.stubbedFsPath) - .should.equal(true) - return done() - }) - }) - - return it('should send the stream in the callback', function(done) { - return this.writer.getStream(this.stubbedFsPath, (err, readStream) => { - readStream.should.equal(this.readStream) - return done() - }) - }) - }) - - return describe('delete file', () => + describe('deleteFile', function() { it('should unlink the file', function(done) { - const error = 'my error' - this.fs.unlink.callsArgWith(1, error) - return this.writer.deleteFile(this.stubbedFsPath, err => { - this.fs.unlink.calledWith(this.stubbedFsPath).should.equal(true) - err.should.equal(error) - return done() + LocalFileWriter.deleteFile(fsPath, err => { + expect(err).not.to.exist + expect(fs.unlink).to.have.been.calledWith(fsPath) + done() }) - })) + }) + + it('should not do anything if called with an empty path', function(done) { + fs.unlink = sinon.stub().yields(new Error('failed to reticulate splines')) + LocalFileWriter.deleteFile(fsPath, err => { + expect(err).to.exist + done() + }) + }) + + it('should not call unlink with an empty path', function(done) { + LocalFileWriter.deleteFile('', err => { + expect(err).not.to.exist + expect(fs.unlink).not.to.have.been.called + done() + }) + }) + }) }) From a216be5fd33166ddbcbde54ff1f34cfc5ac9a141 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 23 Dec 2019 15:58:03 +0000 Subject: [PATCH 40/73] Improve error handling in FileHandler --- services/filestore/app/js/FileHandler.js | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/services/filestore/app/js/FileHandler.js b/services/filestore/app/js/FileHandler.js index 3e102b316b..684a1d2896 100644 --- a/services/filestore/app/js/FileHandler.js +++ b/services/filestore/app/js/FileHandler.js @@ -7,7 +7,12 @@ const FileConverter = require('./FileConverter') const KeyBuilder = require('./KeyBuilder') const async = require('async') const ImageOptimiser = require('./ImageOptimiser') -const { WriteError, ReadError, ConversionError } = require('./Errors') +const { + WriteError, + ReadError, + ConversionError, + NotFoundError +} = require('./Errors') module.exports = { insertFile, @@ -66,16 +71,17 @@ function getDirectorySize(bucket, projectId, callback) { logger.log({ bucket, project_id: projectId }, 'getting project size') PersistorManager.directorySize(bucket, projectId, function(err, size) { if (err) { - logger.err({ bucket, project_id: projectId }, 'error getting size') - err = new ReadError('error getting project size').withCause(err) + return callback( + new ReadError('error getting project size').withCause(err) + ) } - return callback(err, size) + callback(null, size) }) } function _getStandardFile(bucket, key, opts, callback) { PersistorManager.getFileStream(bucket, key, opts, function(err, fileStream) { - if (err && err.name !== 'NotFoundError') { + if (err && !(err instanceof NotFoundError)) { logger.err( { bucket, key, opts: _scrubSecrets(opts) }, 'error getting fileStream' From b6a1ea6a3025145bffda321d30bcefc9b214081a Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Thu, 19 Dec 2019 10:41:41 +0000 Subject: [PATCH 41/73] Cleanup and promisify FileConverter --- services/filestore/app/js/FileConverter.js | 231 ++++++++---------- .../test/unit/js/FileConverterTests.js | 154 ++++++------ 2 files changed, 179 insertions(+), 206 deletions(-) diff --git a/services/filestore/app/js/FileConverter.js b/services/filestore/app/js/FileConverter.js index 04b599728a..0cb5c709fc 100644 --- a/services/filestore/app/js/FileConverter.js +++ b/services/filestore/app/js/FileConverter.js @@ -1,133 +1,116 @@ -/* eslint-disable - camelcase, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const _ = require('underscore') const metrics = require('metrics-sharelatex') const logger = require('logger-sharelatex') -const safe_exec = require('./SafeExec') -const approvedFormats = ['png'] const Settings = require('settings-sharelatex') +const { callbackify } = require('util') -const fourtySeconds = 40 * 1000 +const safeExec = require('./SafeExec').promises +const { ConversionError } = require('./Errors') -const childProcessOpts = { - killSignal: 'SIGTERM', - timeout: fourtySeconds -} +const APPROVED_FORMATS = ['png'] +const FOURTY_SECONDS = 40 * 1000 +const KILL_SIGNAL = 'SIGTERM' module.exports = { - convert(sourcePath, requestedFormat, callback) { - logger.log({ sourcePath, requestedFormat }, 'converting file format') - const timer = new metrics.Timer('imageConvert') - const destPath = `${sourcePath}.${requestedFormat}` - sourcePath = `${sourcePath}[0]` - if (!_.include(approvedFormats, requestedFormat)) { - const err = new Error('invalid format requested') - return callback(err) - } - const width = '600x' - let command = [ - 'convert', - '-define', - `pdf:fit-page=${width}`, - '-flatten', - '-density', - '300', - sourcePath, - destPath - ] - command = Settings.commands.convertCommandPrefix.concat(command) - return safe_exec(command, childProcessOpts, function(err, stdout, stderr) { - timer.done() - if (err != null) { - logger.err( - { err, stderr, sourcePath, requestedFormat, destPath }, - 'something went wrong converting file' - ) - } else { - logger.log( - { sourcePath, requestedFormat, destPath }, - 'finished converting file' - ) - } - return callback(err, destPath) - }) - }, - - thumbnail(sourcePath, callback) { - const destPath = `${sourcePath}.png` - sourcePath = `${sourcePath}[0]` - const width = '260x' - let command = [ - 'convert', - '-flatten', - '-background', - 'white', - '-density', - '300', - '-define', - `pdf:fit-page=${width}`, - sourcePath, - '-resize', - width, - destPath - ] - logger.log({ sourcePath, destPath, command }, 'thumbnail convert file') - command = Settings.commands.convertCommandPrefix.concat(command) - return safe_exec(command, childProcessOpts, function(err, stdout, stderr) { - if (err != null) { - logger.err( - { err, stderr, sourcePath }, - 'something went wrong converting file to thumbnail' - ) - } else { - logger.log({ sourcePath, destPath }, 'finished thumbnailing file') - } - return callback(err, destPath) - }) - }, - - preview(sourcePath, callback) { - logger.log({ sourcePath }, 'preview convert file') - const destPath = `${sourcePath}.png` - sourcePath = `${sourcePath}[0]` - const width = '548x' - let command = [ - 'convert', - '-flatten', - '-background', - 'white', - '-density', - '300', - '-define', - `pdf:fit-page=${width}`, - sourcePath, - '-resize', - width, - destPath - ] - command = Settings.commands.convertCommandPrefix.concat(command) - return safe_exec(command, childProcessOpts, function(err, stdout, stderr) { - if (err != null) { - logger.err( - { err, stderr, sourcePath, destPath }, - 'something went wrong converting file to preview' - ) - } else { - logger.log( - { sourcePath, destPath }, - 'finished converting file to preview' - ) - } - return callback(err, destPath) - }) + convert: callbackify(convert), + thumbnail: callbackify(thumbnail), + preview: callbackify(preview), + promises: { + convert, + thumbnail, + preview } } + +async function convert(sourcePath, requestedFormat) { + const width = '600x' + return _convert(sourcePath, requestedFormat, [ + 'convert', + '-define', + `pdf:fit-page=${width}`, + '-flatten', + '-density', + '300', + `${sourcePath}[0]` + ]) +} + +async function thumbnail(sourcePath) { + const width = '260x' + return convert(sourcePath, 'png', [ + 'convert', + '-flatten', + '-background', + 'white', + '-density', + '300', + '-define', + `pdf:fit-page=${width}`, + `${sourcePath}[0]`, + '-resize', + width + ]) +} + +async function preview(sourcePath) { + const width = '548x' + return convert(sourcePath, 'png', [ + 'convert', + '-flatten', + '-background', + 'white', + '-density', + '300', + '-define', + `pdf:fit-page=${width}`, + `${sourcePath}[0]`, + '-resize', + width + ]) +} + +async function _convert(sourcePath, requestedFormat, command) { + logger.log({ sourcePath, requestedFormat }, 'converting file format') + + if (!APPROVED_FORMATS.includes(requestedFormat)) { + throw new ConversionError({ + message: 'invalid format requested', + info: { format: requestedFormat } + }) + } + + const timer = new metrics.Timer('imageConvert') + const destPath = `${sourcePath}.${requestedFormat}` + + command.push(destPath) + command = Settings.commands.convertCommandPrefix.concat(command) + + try { + await safeExec(command, { + killSignal: KILL_SIGNAL, + timeout: FOURTY_SECONDS + }) + } catch (err) { + logger.err( + { + err, + stderr: err.stderr, + command, + sourcePath, + requestedFormat, + destPath + }, + 'something went wrong converting file' + ) + throw new ConversionError({ + message: 'something went wrong converting file', + info: { stderr: err.stderr, sourcePath, requestedFormat, destPath } + }).withCause(err) + } + + timer.done() + logger.log( + { sourcePath, requestedFormat, destPath }, + 'finished converting file' + ) + return destPath +} diff --git a/services/filestore/test/unit/js/FileConverterTests.js b/services/filestore/test/unit/js/FileConverterTests.js index 89faacd1d1..7a7ba12ee2 100644 --- a/services/filestore/test/unit/js/FileConverterTests.js +++ b/services/filestore/test/unit/js/FileConverterTests.js @@ -1,29 +1,30 @@ -/* eslint-disable - handle-callback-err, - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const { assert } = require('chai') const sinon = require('sinon') const chai = require('chai') -const should = chai.should() const { expect } = chai -const modulePath = '../../../app/js/FileConverter.js' const SandboxedModule = require('sandboxed-module') +const modulePath = '../../../app/js/FileConverter.js' + describe('FileConverter', function() { + let SafeExec, FileConverter + const sourcePath = '/data/wombat.eps' + const destPath = '/tmp/dest.png' + const format = 'png' + const errorMessage = 'guru meditation error' + const Settings = { + commands: { + convertCommandPrefix: [] + } + } + beforeEach(function() { - this.safe_exec = sinon.stub() - this.converter = SandboxedModule.require(modulePath, { + SafeExec = { + promises: sinon.stub().resolves(destPath) + } + + FileConverter = SandboxedModule.require(modulePath, { requires: { - './SafeExec': this.safe_exec, + './SafeExec': SafeExec, 'logger-sharelatex': { log() {}, err() {} @@ -32,86 +33,75 @@ describe('FileConverter', function() { inc: sinon.stub(), Timer: sinon.stub().returns({ done: sinon.stub() }) }, - 'settings-sharelatex': (this.Settings = { - commands: { - convertCommandPrefix: [] - } - }) + 'settings-sharelatex': Settings } }) - - this.sourcePath = '/this/path/here.eps' - this.format = 'png' - return (this.error = 'Error') }) describe('convert', function() { - it('should convert the source to the requested format', function(done) { - this.safe_exec.callsArgWith(2) - return this.converter.convert(this.sourcePath, this.format, err => { - const args = this.safe_exec.args[0][0] - args.indexOf(`${this.sourcePath}[0]`).should.not.equal(-1) - args.indexOf(`${this.sourcePath}.${this.format}`).should.not.equal(-1) - return done() - }) + it('should convert the source to the requested format', async function() { + await FileConverter.promises.convert(sourcePath, format) + const args = SafeExec.promises.args[0][0] + expect(args).to.include(`${sourcePath}[0]`) + expect(args).to.include(`${sourcePath}.${format}`) }) - it('should return the dest path', function(done) { - this.safe_exec.callsArgWith(2) - return this.converter.convert( - this.sourcePath, - this.format, - (err, destPath) => { - destPath.should.equal(`${this.sourcePath}.${this.format}`) - return done() - } - ) + it('should return the dest path', async function() { + const destPath = await FileConverter.promises.convert(sourcePath, format) + destPath.should.equal(`${sourcePath}.${format}`) }) - it('should return the error from convert', function(done) { - this.safe_exec.callsArgWith(2, this.error) - return this.converter.convert(this.sourcePath, this.format, err => { - err.should.equal(this.error) - return done() - }) + it('should wrap the error from convert', async function() { + SafeExec.promises.rejects(errorMessage) + try { + await FileConverter.promises.convert(sourcePath, format) + expect('error should have been thrown').not.to.exist + } catch (err) { + expect(err.name).to.equal('ConversionError') + expect(err.cause.toString()).to.equal(errorMessage) + } }) - it('should not accapt an non aproved format', function(done) { - this.safe_exec.callsArgWith(2) - return this.converter.convert(this.sourcePath, 'ahhhhh', err => { - expect(err).to.exist - return done() - }) + it('should not accept an non approved format', async function() { + try { + await FileConverter.promises.convert(sourcePath, 'potato') + expect('error should have been thrown').not.to.exist + } catch (err) { + expect(err.name).to.equal('ConversionError') + } }) - return it('should prefix the command with Settings.commands.convertCommandPrefix', function(done) { - this.safe_exec.callsArgWith(2) - this.Settings.commands.convertCommandPrefix = ['nice'] - return this.converter.convert(this.sourcePath, this.format, err => { - const command = this.safe_exec.args[0][0] - command[0].should.equal('nice') - return done() + it('should prefix the command with Settings.commands.convertCommandPrefix', async function() { + Settings.commands.convertCommandPrefix = ['nice'] + await FileConverter.promises.convert(sourcePath, format) + }) + + it('should convert the file when called as a callback', function(done) { + FileConverter.convert(sourcePath, format, (err, destPath) => { + expect(err).not.to.exist + destPath.should.equal(`${sourcePath}.${format}`) + + const args = SafeExec.promises.args[0][0] + expect(args).to.include(`${sourcePath}[0]`) + expect(args).to.include(`${sourcePath}.${format}`) + done() }) }) }) - describe('thumbnail', () => - it('should call converter resize with args', function(done) { - this.safe_exec.callsArgWith(2) - return this.converter.thumbnail(this.sourcePath, err => { - const args = this.safe_exec.args[0][0] - args.indexOf(`${this.sourcePath}[0]`).should.not.equal(-1) - return done() - }) - })) + describe('thumbnail', function() { + it('should call converter resize with args', async function() { + await FileConverter.promises.thumbnail(sourcePath) + const args = SafeExec.promises.args[0][0] + expect(args).to.include(`${sourcePath}[0]`) + }) + }) - return describe('preview', () => - it('should call converter resize with args', function(done) { - this.safe_exec.callsArgWith(2) - return this.converter.preview(this.sourcePath, err => { - const args = this.safe_exec.args[0][0] - args.indexOf(`${this.sourcePath}[0]`).should.not.equal(-1) - return done() - }) - })) + describe('preview', function() { + it('should call converter resize with args', async function() { + await FileConverter.promises.preview(sourcePath) + const args = SafeExec.promises.args[0][0] + expect(args).to.include(`${sourcePath}[0]`) + }) + }) }) From ac2d05ecb3d6da0bca8064b01b2bc0f46006af58 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 23 Dec 2019 16:03:01 +0000 Subject: [PATCH 42/73] Remove unnecessary logging from FileConverter --- services/filestore/app/js/FileConverter.js | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/services/filestore/app/js/FileConverter.js b/services/filestore/app/js/FileConverter.js index 0cb5c709fc..9338b289fb 100644 --- a/services/filestore/app/js/FileConverter.js +++ b/services/filestore/app/js/FileConverter.js @@ -90,17 +90,6 @@ async function _convert(sourcePath, requestedFormat, command) { timeout: FOURTY_SECONDS }) } catch (err) { - logger.err( - { - err, - stderr: err.stderr, - command, - sourcePath, - requestedFormat, - destPath - }, - 'something went wrong converting file' - ) throw new ConversionError({ message: 'something went wrong converting file', info: { stderr: err.stderr, sourcePath, requestedFormat, destPath } From 42adc59d018e906ddf662ea9c50ec970a325122a Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Thu, 19 Dec 2019 15:36:48 +0000 Subject: [PATCH 43/73] Clean up and promisify health-check controller and KeyBuilder --- services/filestore/app.js | 30 ++-- services/filestore/app/js/Errors.js | 4 +- .../filestore/app/js/HealthCheckController.js | 114 +++++++------- services/filestore/app/js/KeyBuilder.js | 139 +++++++++--------- 4 files changed, 144 insertions(+), 143 deletions(-) diff --git a/services/filestore/app.js b/services/filestore/app.js index c6c11e152a..9e76107ea6 100644 --- a/services/filestore/app.js +++ b/services/filestore/app.js @@ -89,82 +89,82 @@ Metrics.injectMetricsRoute(app) app.head( '/project/:project_id/file/:file_id', - keyBuilder.userFileKey, + keyBuilder.userFileKeyMiddleware, fileController.getFileHead ) app.get( '/project/:project_id/file/:file_id', - keyBuilder.userFileKey, + keyBuilder.userFileKeyMiddleware, fileController.getFile ) app.post( '/project/:project_id/file/:file_id', - keyBuilder.userFileKey, + keyBuilder.userFileKeyMiddleware, fileController.insertFile ) app.put( '/project/:project_id/file/:file_id', - keyBuilder.userFileKey, + keyBuilder.userFileKeyMiddleware, bodyParser.json(), fileController.copyFile ) app.del( '/project/:project_id/file/:file_id', - keyBuilder.userFileKey, + keyBuilder.userFileKeyMiddleware, fileController.deleteFile ) app.head( '/template/:template_id/v/:version/:format', - keyBuilder.templateFileKey, + keyBuilder.templateFileKeyMiddleware, fileController.getFileHead ) app.get( '/template/:template_id/v/:version/:format', - keyBuilder.templateFileKey, + keyBuilder.templateFileKeyMiddleware, fileController.getFile ) app.get( '/template/:template_id/v/:version/:format/:sub_type', - keyBuilder.templateFileKey, + keyBuilder.templateFileKeyMiddleware, fileController.getFile ) app.post( '/template/:template_id/v/:version/:format', - keyBuilder.templateFileKey, + keyBuilder.templateFileKeyMiddleware, fileController.insertFile ) app.head( '/project/:project_id/public/:public_file_id', - keyBuilder.publicFileKey, + keyBuilder.publicFileKeyMiddleware, fileController.getFileHead ) app.get( '/project/:project_id/public/:public_file_id', - keyBuilder.publicFileKey, + keyBuilder.publicFileKeyMiddleware, fileController.getFile ) app.post( '/project/:project_id/public/:public_file_id', - keyBuilder.publicFileKey, + keyBuilder.publicFileKeyMiddleware, fileController.insertFile ) app.put( '/project/:project_id/public/:public_file_id', - keyBuilder.publicFileKey, + keyBuilder.publicFileKeyMiddleware, bodyParser.json(), fileController.copyFile ) app.del( '/project/:project_id/public/:public_file_id', - keyBuilder.publicFileKey, + keyBuilder.publicFileKeyMiddleware, fileController.deleteFile ) app.get( '/project/:project_id/size', - keyBuilder.publicProjectKey, + keyBuilder.publicProjectKeyMiddleware, fileController.directorySize ) diff --git a/services/filestore/app/js/Errors.js b/services/filestore/app/js/Errors.js index 57dbdbe522..65af6dc056 100644 --- a/services/filestore/app/js/Errors.js +++ b/services/filestore/app/js/Errors.js @@ -20,6 +20,7 @@ class BackwardCompatibleError extends OError { class NotFoundError extends BackwardCompatibleError {} class WriteError extends BackwardCompatibleError {} class ReadError extends BackwardCompatibleError {} +class HealthCheckError extends BackwardCompatibleError {} class ConversionsDisabledError extends BackwardCompatibleError {} class ConversionError extends BackwardCompatibleError {} @@ -44,5 +45,6 @@ module.exports = { ConversionsDisabledError, WriteError, ReadError, - ConversionError + ConversionError, + HealthCheckError } diff --git a/services/filestore/app/js/HealthCheckController.js b/services/filestore/app/js/HealthCheckController.js index eecadb00d9..8d6e35b783 100644 --- a/services/filestore/app/js/HealthCheckController.js +++ b/services/filestore/app/js/HealthCheckController.js @@ -1,80 +1,72 @@ -// TODO: This file was created by bulk-decaffeinate. -// Sanity-check the conversion and remove this comment. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ const fs = require('fs-extra') const path = require('path') -const async = require('async') -const fileConverter = require('./FileConverter') -const keyBuilder = require('./KeyBuilder') -const fileController = require('./FileController') const logger = require('logger-sharelatex') -const settings = require('settings-sharelatex') +const Settings = require('settings-sharelatex') const streamBuffers = require('stream-buffers') -const _ = require('underscore') +const { promisify } = require('util') +const Stream = require('stream') -const checkCanStoreFiles = function(callback) { - callback = _.once(callback) - const req = { params: {}, query: {}, headers: {} } - req.params.project_id = settings.health_check.project_id - req.params.file_id = settings.health_check.file_id - const myWritableStreamBuffer = new streamBuffers.WritableStreamBuffer({ +const pipeline = promisify(Stream.pipeline) +const fsCopy = promisify(fs.copy) +const fsUnlink = promisify(fs.unlink) + +const { HealthCheckError } = require('./Errors') +const FileConverter = require('./FileConverter').promises +const FileHandler = require('./FileHandler').promises + +async function checkCanGetFiles() { + if (!Settings.health_check) { + return + } + + const projectId = Settings.health_check.project_id + const fileId = Settings.health_check.file_id + const key = `${projectId}/${fileId}` + const bucket = Settings.filestore.stores.user_files + + const buffer = new streamBuffers.WritableStreamBuffer({ initialSize: 100 }) - const res = { - send(code) { - if (code !== 200) { - return callback(new Error(`non-200 code from getFile: ${code}`)) - } - } + + const sourceStream = await FileHandler.getFile(bucket, key, {}) + try { + await pipeline(sourceStream, buffer) + } catch (err) { + throw new HealthCheckError('failed to get health-check file').withCause(err) + } + + if (!buffer.size()) { + throw new HealthCheckError('no bytes written to download stream') } - myWritableStreamBuffer.send = res.send - return keyBuilder.userFileKey(req, res, function() { - fileController.getFile(req, myWritableStreamBuffer) - return myWritableStreamBuffer.on('close', function() { - if (myWritableStreamBuffer.size() > 0) { - return callback() - } else { - const err = 'no data in write stream buffer for health check' - logger.err({ err }, 'error performing health check') - return callback(err) - } - }) - }) } -const checkFileConvert = function(callback) { - if (!settings.enableConversions) { - return callback() +async function checkFileConvert() { + if (!Settings.enableConversions) { + return + } + + const imgPath = path.join(Settings.path.uploadFolder, '/tiny.pdf') + + let resultPath + try { + await fsCopy('./tiny.pdf', imgPath) + resultPath = await FileConverter.thumbnail(imgPath) + } finally { + if (resultPath) { + await fsUnlink(resultPath) + } + await fsUnlink(imgPath) } - const imgPath = path.join(settings.path.uploadFolder, '/tiny.pdf') - return async.waterfall( - [ - cb => fs.copy('./tiny.pdf', imgPath, cb), - cb => fileConverter.thumbnail(imgPath, cb), - (resultPath, cb) => fs.unlink(resultPath, cb), - cb => fs.unlink(imgPath, cb) - ], - callback - ) } module.exports = { check(req, res) { logger.log({}, 'performing health check') - return async.parallel([checkFileConvert, checkCanStoreFiles], function( - err - ) { - if (err != null) { + Promise.all([checkCanGetFiles(), checkFileConvert()]) + .then(() => res.send(200)) + .catch(err => { logger.err({ err }, 'Health check: error running') - return res.send(500) - } else { - return res.send(200) - } - }) + res.send(500) + }) } } diff --git a/services/filestore/app/js/KeyBuilder.js b/services/filestore/app/js/KeyBuilder.js index 110900c991..8de7c0be2a 100644 --- a/services/filestore/app/js/KeyBuilder.js +++ b/services/filestore/app/js/KeyBuilder.js @@ -1,71 +1,78 @@ -/* eslint-disable - camelcase, - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ const settings = require('settings-sharelatex') module.exports = { - getConvertedFolderKey(key) { - return (key = `${key}-converted-cache/`) - }, - - addCachingToKey(key, opts) { - key = this.getConvertedFolderKey(key) - if (opts.format != null && opts.style == null) { - key = `${key}format-${opts.format}` - } - if (opts.style != null && opts.format == null) { - key = `${key}style-${opts.style}` - } - if (opts.style != null && opts.format != null) { - key = `${key}format-${opts.format}-style-${opts.style}` - } - return key - }, - - userFileKey(req, res, next) { - const { project_id, file_id } = req.params - req.key = `${project_id}/${file_id}` - req.bucket = settings.filestore.stores.user_files - return next() - }, - - publicFileKey(req, res, next) { - const { project_id, public_file_id } = req.params - if (settings.filestore.stores.public_files == null) { - return res.status(501).send('public files not available') - } else { - req.key = `${project_id}/${public_file_id}` - req.bucket = settings.filestore.stores.public_files - return next() - } - }, - - templateFileKey(req, res, next) { - const { template_id, format, version, sub_type } = req.params - req.key = `${template_id}/v/${version}/${format}` - if (sub_type != null) { - req.key = `${req.key}/${sub_type}` - } - req.bucket = settings.filestore.stores.template_files - req.version = version - const opts = req.query - return next() - }, - - publicProjectKey(req, res, next) { - const { project_id } = req.params - req.project_id = project_id - req.bucket = settings.filestore.stores.user_files - return next() + getConvertedFolderKey, + addCachingToKey, + userFileKeyMiddleware, + publicFileKeyMiddleware, + publicProjectKeyMiddleware, + templateFileKeyMiddleware +} + +function getConvertedFolderKey(key) { + return `${key}-converted-cache/` +} + +function addCachingToKey(key, opts) { + key = this.getConvertedFolderKey(key) + + if (opts.format && !opts.style) { + key = `${key}format-${opts.format}` + } + if (opts.style && !opts.format) { + key = `${key}style-${opts.style}` + } + if (opts.style && opts.format) { + key = `${key}format-${opts.format}-style-${opts.style}` + } + + return key +} + +function userFileKeyMiddleware(req, res, next) { + const { project_id: projectId, file_id: fileId } = req.params + req.key = `${projectId}/${fileId}` + req.bucket = settings.filestore.stores.user_files + next() +} + +function publicFileKeyMiddleware(req, res, next) { + if (settings.filestore.stores.public_files == null) { + return res.status(501).send('public files not available') + } + + const { project_id: projectId, public_file_id: publicFileId } = req.params + req.key = `${projectId}/${publicFileId}` + req.bucket = settings.filestore.stores.public_files + + next() +} + +function templateFileKeyMiddleware(req, res, next) { + const { + template_id: templateId, + format, + version, + sub_type: subType + } = req.params + + req.key = `${templateId}/v/${version}/${format}` + + if (subType) { + req.key = `${req.key}/${subType}` } + + req.bucket = settings.filestore.stores.template_files + req.version = version + + next() +} + +function publicProjectKeyMiddleware(req, res, next) { + const { project_id: projectId } = req.params + + req.project_id = projectId + req.bucket = settings.filestore.stores.user_files + + next() } From 006f84abebdd0117ae9d7fc9384d5449caacdeaa Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Thu, 19 Dec 2019 16:56:03 +0000 Subject: [PATCH 44/73] Decaf and promisify ImageOptimiser --- services/filestore/app/js/ImageOptimiser.js | 73 ++++++++-------- .../test/unit/js/ImageOptimiserTests.js | 84 +++++++------------ 2 files changed, 66 insertions(+), 91 deletions(-) diff --git a/services/filestore/app/js/ImageOptimiser.js b/services/filestore/app/js/ImageOptimiser.js index 99f1eb21f6..555e6334bd 100644 --- a/services/filestore/app/js/ImageOptimiser.js +++ b/services/filestore/app/js/ImageOptimiser.js @@ -1,44 +1,41 @@ -/* eslint-disable - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const { exec } = require('child_process') const logger = require('logger-sharelatex') -const Settings = require('settings-sharelatex') +const metrics = require('metrics-sharelatex') +const { callbackify } = require('util') +const safeExec = require('./SafeExec').promises module.exports = { - compressPng(localPath, callback) { - const startTime = new Date() - logger.log({ localPath }, 'optimising png path') - const args = `optipng ${localPath}` - const opts = { - timeout: 30 * 1000, - killSignal: 'SIGKILL' + compressPng: callbackify(compressPng), + promises: { + compressPng + } +} + +async function compressPng(localPath, callback) { + const timer = new metrics.Timer('compressPng') + logger.log({ localPath }, 'optimising png path') + + const args = ['optipng', localPath] + const opts = { + timeout: 30 * 1000, + killSignal: 'SIGKILL' + } + + try { + await safeExec(args, opts) + timer.done() + logger.log({ localPath }, 'finished compressing png') + } catch (err) { + if (err.code === 'SIGKILL') { + logger.warn( + { err, stderr: err.stderr, localPath }, + 'optimiser timeout reached' + ) + } else { + logger.err( + { err, stderr: err.stderr, localPath }, + 'something went wrong compressing png' + ) + throw err } - if (!Settings.enableConversions) { - const error = new Error('Image conversions are disabled') - return callback(error) - } - return exec(args, opts, function(err, stdout, stderr) { - if (err != null && err.signal === 'SIGKILL') { - logger.warn({ err, stderr, localPath }, 'optimiser timeout reached') - err = null - } else if (err != null) { - logger.err( - { err, stderr, localPath }, - 'something went wrong converting compressPng' - ) - } else { - logger.log({ localPath }, 'finished compressPng file') - } - return callback(err) - }) } } diff --git a/services/filestore/test/unit/js/ImageOptimiserTests.js b/services/filestore/test/unit/js/ImageOptimiserTests.js index 6fdb09f89d..7940767fdf 100644 --- a/services/filestore/test/unit/js/ImageOptimiserTests.js +++ b/services/filestore/test/unit/js/ImageOptimiserTests.js @@ -1,81 +1,59 @@ -/* eslint-disable - handle-callback-err, - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const { assert } = require('chai') const sinon = require('sinon') const chai = require('chai') -const should = chai.should() const { expect } = chai const modulePath = '../../../app/js/ImageOptimiser.js' +const { FailedCommandError } = require('../../../app/js/Errors') const SandboxedModule = require('sandboxed-module') describe('ImageOptimiser', function() { + let ImageOptimiser, SafeExec + const sourcePath = '/wombat/potato.eps' + beforeEach(function() { - this.child_process = { exec: sinon.stub() } - this.settings = { enableConversions: true } - this.optimiser = SandboxedModule.require(modulePath, { + SafeExec = { + promises: sinon.stub().resolves() + } + ImageOptimiser = SandboxedModule.require(modulePath, { requires: { - child_process: this.child_process, + './SafeExec': SafeExec, 'logger-sharelatex': { log() {}, err() {}, warn() {} - }, - 'settings-sharelatex': this.settings + } } }) - - this.sourcePath = '/this/path/here.eps' - return (this.error = 'Error') }) describe('compressPng', function() { - it('convert the file', function(done) { - this.child_process.exec.callsArgWith(2) - return this.optimiser.compressPng(this.sourcePath, err => { - const args = this.child_process.exec.args[0][0] - args.should.equal(`optipng ${this.sourcePath}`) - return done() + it('should convert the file', function(done) { + ImageOptimiser.compressPng(sourcePath, err => { + expect(err).not.to.exist + expect(SafeExec.promises).to.have.been.calledWith([ + 'optipng', + sourcePath + ]) + done() }) }) - return it('should return the error', function(done) { - this.child_process.exec.callsArgWith(2, this.error) - return this.optimiser.compressPng(this.sourcePath, err => { - err.should.equal(this.error) - return done() + it('should return the error', function(done) { + SafeExec.promises.rejects('wombat herding failure') + ImageOptimiser.compressPng(sourcePath, err => { + expect(err.toString()).to.equal('wombat herding failure') + done() }) }) }) - describe('when enableConversions is disabled', () => - it('should produce an error', function(done) { - this.settings.enableConversions = false - this.child_process.exec.callsArgWith(2) - return this.optimiser.compressPng(this.sourcePath, err => { - this.child_process.exec.called.should.equal(false) - expect(err).to.exist - return done() - }) - })) - - return describe('when optimiser is sigkilled', () => + describe('when optimiser is sigkilled', function() { it('should not produce an error', function(done) { - this.error = new Error('woops') - this.error.signal = 'SIGKILL' - this.child_process.exec.callsArgWith(2, this.error) - return this.optimiser.compressPng(this.sourcePath, err => { - expect(err).to.equal(null) - return done() + const error = new FailedCommandError('', 'SIGKILL', '', '') + SafeExec.promises.rejects(error) + ImageOptimiser.compressPng(sourcePath, err => { + expect(err).not.to.exist + done() }) - })) + }) + }) }) From a8158d6c8c761ad5c4b616d7aa815fb7a315fbc1 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 23 Dec 2019 13:36:12 +0000 Subject: [PATCH 45/73] Cleanup and promisify acceptance tests. Run tests for all backends. --- services/filestore/npm-shrinkwrap.json | 158 +++++++-- services/filestore/package.json | 5 +- .../test/acceptance/js/FilestoreApp.js | 193 ++++++----- .../test/acceptance/js/FilestoreTests.js | 299 ++++++++++++++++ .../test/acceptance/js/SendingFileTest.js | 326 ------------------ 5 files changed, 521 insertions(+), 460 deletions(-) create mode 100644 services/filestore/test/acceptance/js/FilestoreTests.js delete mode 100644 services/filestore/test/acceptance/js/SendingFileTest.js diff --git a/services/filestore/npm-shrinkwrap.json b/services/filestore/npm-shrinkwrap.json index 44f5ec3263..fa498f3f1b 100644 --- a/services/filestore/npm-shrinkwrap.json +++ b/services/filestore/npm-shrinkwrap.json @@ -1014,6 +1014,15 @@ "type-detect": "^4.0.5" } }, + "chai-as-promised": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz", + "integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==", + "dev": true, + "requires": { + "check-error": "^1.0.2" + } + }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -1284,6 +1293,12 @@ "integrity": "sha1-gAwN0eCov7yVg1wgKtIg/jF+WhI=", "dev": true }, + "disrequire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/disrequire/-/disrequire-1.1.0.tgz", + "integrity": "sha512-c3lya+wBcnfNipVE7XQC85J6Fty9XWsbNrUub8XT1Qk3mwO6f8tR7P6Ah3X09A3HTQ1biwjcwTLFkGlEejUzUw==", + "dev": true + }, "dlv": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", @@ -4446,53 +4461,115 @@ "dev": true }, "request": { - "version": "2.14.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.14.0.tgz", - "integrity": "sha1-DYrLsLFMGrguAAt9OB+oyA0afYg=", + "version": "2.88.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", "requires": { - "form-data": "~0.0.3", - "mime": "~1.2.7" + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.0", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.4.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" }, "dependencies": { - "form-data": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-0.0.7.tgz", - "integrity": "sha1-chEYKiaiZs45cQ3IvEqBtwQIWb4=", + "caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" + }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", "requires": { - "async": "~0.1.9", - "combined-stream": "~0.0.4", - "mime": "~1.2.2" - }, - "dependencies": { - "async": { - "version": "0.1.22", - "resolved": "https://registry.npmjs.org/async/-/async-0.1.22.tgz", - "integrity": "sha1-D8GqoIig4+8Ovi2IMbqw3PiEUGE=" - }, - "combined-stream": { - "version": "0.0.4", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.4.tgz", - "integrity": "sha1-LRpDNH2+lRWkonlnMuW4hHOECyI=", - "requires": { - "delayed-stream": "0.0.5" - }, - "dependencies": { - "delayed-stream": { - "version": "0.0.5", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz", - "integrity": "sha1-1LH0OpPoKW3+AmlPRoC8N6MTxz8=" - } - } - } + "delayed-stream": "~1.0.0" } }, - "mime": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.9.tgz", - "integrity": "sha1-AJzUCGe9Nd5SGzuWbwTi+NTRPQk=" + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" + }, + "forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" + }, + "form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + }, + "oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" + }, + "safe-buffer": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz", + "integrity": "sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg==" + }, + "tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "requires": { + "safe-buffer": "^5.0.1" + } + }, + "uuid": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.3.tgz", + "integrity": "sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ==" } } }, + "request-promise-core": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.3.tgz", + "integrity": "sha512-QIs2+ArIGQVp5ZYbWD5ZLCY29D5CfWizP8eWnm8FoGD1TX61veauETVQbrV60662V0oFBkrDOuaBI8XgtuyYAQ==", + "requires": { + "lodash": "^4.17.15" + }, + "dependencies": { + "lodash": { + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" + } + } + }, + "request-promise-native": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/request-promise-native/-/request-promise-native-1.0.8.tgz", + "integrity": "sha512-dapwLGqkHtwL5AEbfenuzjTYg35Jd6KPytsC2/TLkVMz8rm+tNt72MGUWT1RP/aYawMpN6HqbNGBQaRcBtjQMQ==", + "requires": { + "request-promise-core": "1.1.3", + "stealthy-require": "^1.1.1", + "tough-cookie": "^2.3.3" + } + }, "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -4878,6 +4955,11 @@ "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" }, + "stealthy-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", + "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=" + }, "stream-browserify": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.1.tgz", diff --git a/services/filestore/package.json b/services/filestore/package.json index d39d1027be..9515c1850c 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -36,7 +36,8 @@ "pngcrush": "0.0.3", "range-parser": "^1.0.2", "recluster": "^0.3.7", - "request": "2.14.0", + "request": "^2.88.0", + "request-promise-native": "^1.0.8", "response": "0.14.0", "rimraf": "2.2.8", "settings-sharelatex": "^1.1.0", @@ -48,6 +49,8 @@ "babel-eslint": "^10.0.3", "bunyan": "^1.3.5", "chai": "4.2.0", + "chai-as-promised": "^7.1.1", + "disrequire": "^1.1.0", "eslint": "^6.4.0", "eslint-config-prettier": "^6.7.0", "eslint-config-standard": "^14.1.0", diff --git a/services/filestore/test/acceptance/js/FilestoreApp.js b/services/filestore/test/acceptance/js/FilestoreApp.js index 3a5103d5f6..4035262cbc 100644 --- a/services/filestore/test/acceptance/js/FilestoreApp.js +++ b/services/filestore/test/acceptance/js/FilestoreApp.js @@ -1,109 +1,112 @@ -/* eslint-disable - handle-callback-err, - standard/no-callback-literal, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS103: Rewrite code to no longer use __guard__ - * DS205: Consider reworking code to avoid use of IIFEs - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const app = require('../../../app') -require('logger-sharelatex').logger.level('info') const logger = require('logger-sharelatex') const Settings = require('settings-sharelatex') +const fs = require('fs') +const Path = require('path') const request = require('request') +const { promisify } = require('util') +const disrequire = require('disrequire') const S3_TRIES = 30 -module.exports = { - running: false, - initing: false, - callbacks: [], - ensureRunning(callback) { - if (callback == null) { - callback = function(error) {} - } - if (this.running) { - return callback() - } else if (this.initing) { - return this.callbacks.push(callback) - } else { - this.initing = true - this.callbacks.push(callback) - return app.listen( - __guard__( - Settings.internal != null ? Settings.internal.filestore : undefined, - x => x.port - ), - 'localhost', - error => { - if (error != null) { - throw error - } - this.running = true - logger.log('filestore running in dev mode') +logger.logger.level('info') - return (() => { - const result = [] - for (callback of Array.from(this.callbacks)) { - result.push(callback()) - } - return result - })() +const fsReaddir = promisify(fs.readdir) + +class FilestoreApp { + constructor() { + this.running = false + this.initing = false + } + + async runServer() { + if (this.running) { + return + } + + if (this.initing) { + return this.waitForInit() + } + this.initing = true + + this.app = await FilestoreApp.requireApp() + + await new Promise((resolve, reject) => { + this.server = this.app.listen( + Settings.internal.filestore.port, + 'localhost', + err => { + if (err) { + return reject(err) + } + resolve() } ) - } - }, + }) - waitForS3(callback, tries) { - if ( - !(Settings.filestore.s3 != null - ? Settings.filestore.s3.endpoint - : undefined) - ) { - return callback() - } - if (!tries) { - tries = 1 - } - - return request.get( - `${Settings.filestore.s3.endpoint}/`, - (err, response) => { - console.log( - err, - response != null ? response.statusCode : undefined, - tries - ) - if ( - !err && - [200, 404].includes( - response != null ? response.statusCode : undefined - ) - ) { - return callback() - } - - if (tries === S3_TRIES) { - return callback('timed out waiting for S3') - } - - return setTimeout(() => { - return this.waitForS3(callback, tries + 1) - }, 1000) + if (Settings.filestore.backend === 's3') { + try { + await FilestoreApp.waitForS3() + } catch (err) { + await this.stop() + throw err } - ) + } + + this.initing = false + } + + async waitForInit() { + while (this.initing) { + await new Promise(resolve => setTimeout(resolve, 1000)) + } + } + + async stop() { + if (this.server) { + await new Promise(resolve => { + this.server.close(resolve) + }) + delete this.server + } + } + + static async waitForS3() { + let tries = 0 + if (!Settings.filestore.s3.endpoint) { + return + } + + let s3Available = false + + while (tries < S3_TRIES && !s3Available) { + try { + const response = await promisify(request.get)( + `${Settings.filestore.s3.endpoint}/` + ) + if ([200, 404].includes(response.statusCode)) { + s3Available = true + } + } catch (err) { + } finally { + tries++ + if (!s3Available) { + await new Promise(resolve => setTimeout(resolve, 1000)) + } + } + } + } + + static async requireApp() { + // unload the app, as we may be doing this on multiple runs with + // different settings, which affect startup in some cases + const files = await fsReaddir(Path.resolve(__dirname, '../../../app/js')) + files.forEach(file => { + disrequire(Path.resolve(__dirname, '../../../app/js', file)) + }) + disrequire(Path.resolve(__dirname, '../../../app')) + + return require('../../../app') } } -function __guard__(value, transform) { - return typeof value !== 'undefined' && value !== null - ? transform(value) - : undefined -} +module.exports = FilestoreApp diff --git a/services/filestore/test/acceptance/js/FilestoreTests.js b/services/filestore/test/acceptance/js/FilestoreTests.js new file mode 100644 index 0000000000..3315569a24 --- /dev/null +++ b/services/filestore/test/acceptance/js/FilestoreTests.js @@ -0,0 +1,299 @@ +const chai = require('chai') +const { expect } = chai +const fs = require('fs') +const Settings = require('settings-sharelatex') +const Path = require('path') +const FilestoreApp = require('./FilestoreApp') +const rp = require('request-promise-native').defaults({ + resolveWithFullResponse: true +}) +const Stream = require('stream') +const request = require('request') +const { promisify } = require('util') +chai.use(require('chai-as-promised')) + +const fsWriteFile = promisify(fs.writeFile) +const fsStat = promisify(fs.stat) +const pipeline = promisify(Stream.pipeline) + +async function getMetric(filestoreUrl, metric) { + const res = await rp.get(`${filestoreUrl}/metrics`) + expect(res.statusCode).to.equal(200) + const metricRegex = new RegExp(`^${metric}{[^}]+} ([0-9]+)$`, 'm') + const found = metricRegex.exec(res.body) + return parseInt(found ? found[1] : 0) || 0 +} + +// store settings for multiple backends, so that we can test each one. +// fs will always be available - add others if they are configured +const BackendSettings = { + FSPersistor: { + backend: 'fs', + stores: { + user_files: Path.resolve(__dirname, '../../../user_files'), + public_files: Path.resolve(__dirname, '../../../public_files'), + template_files: Path.resolve(__dirname, '../../../template_files') + } + } +} + +if (process.env.AWS_ACCESS_KEY_ID) { + BackendSettings.S3Persistor = { + backend: 's3', + s3: { + key: process.env.AWS_ACCESS_KEY_ID, + secret: process.env.AWS_SECRET_ACCESS_KEY, + endpoint: process.env.AWS_S3_ENDPOINT + }, + stores: { + user_files: process.env.AWS_S3_USER_FILES_BUCKET_NAME, + template_files: process.env.AWS_S3_TEMPLATE_FILES_BUCKET_NAME, + public_files: process.env.AWS_S3_PUBLIC_FILES_BUCKET_NAME + } + } +} + +describe('Filestore', function() { + this.timeout(1000 * 10) + const filestoreUrl = `http://localhost:${Settings.internal.filestore.port}` + + // redefine the test suite for every available backend + Object.keys(BackendSettings).forEach(backend => { + describe(backend, function() { + let app, previousEgress, previousIngress + + before(async function() { + // create the app with the relevant filestore settings + Settings.filestore = BackendSettings[backend] + app = new FilestoreApp() + await app.runServer() + }) + + after(async function() { + return app.stop() + }) + + beforeEach(async function() { + // retrieve previous metrics from the app + if (Settings.filestore.backend === 's3') { + ;[previousEgress, previousIngress] = await Promise.all([ + getMetric(filestoreUrl, 's3_egress'), + getMetric(filestoreUrl, 's3_ingress') + ]) + } + }) + + it('should send a 200 for the status endpoint', async function() { + const response = await rp(`${filestoreUrl}/status`) + expect(response.statusCode).to.equal(200) + expect(response.body).to.contain('filestore') + expect(response.body).to.contain('up') + }) + + it('should send a 200 for the health-check endpoint', async function() { + const response = await rp(`${filestoreUrl}/health_check`) + expect(response.statusCode).to.equal(200) + expect(response.body).to.equal('OK') + }) + + describe('with a file on the server', function() { + let fileId, fileUrl + + const localFileReadPath = + '/tmp/filestore_acceptance_tests_file_read.txt' + const constantFileContent = [ + 'hello world', + `line 2 goes here ${Math.random()}`, + 'there are 3 lines in all' + ].join('\n') + + before(async function() { + await fsWriteFile(localFileReadPath, constantFileContent) + }) + + beforeEach(async function() { + fileId = Math.random() + fileUrl = `${filestoreUrl}/project/acceptance_tests/file/${fileId}` + + const writeStream = request.post(fileUrl) + const readStream = fs.createReadStream(localFileReadPath) + // consume the result to ensure the http request has been fully processed + const resultStream = fs.createWriteStream('/dev/null') + await pipeline(readStream, writeStream, resultStream) + }) + + it('should return 404 for a non-existant id', async function() { + const options = { uri: fileUrl + '___this_is_clearly_wrong___' } + await expect( + rp.get(options) + ).to.eventually.be.rejected.and.have.property('statusCode', 404) + }) + + it('should return the file size on a HEAD request', async function() { + const expectedLength = Buffer.byteLength(constantFileContent) + const res = await rp.head(fileUrl) + expect(res.statusCode).to.equal(200) + expect(res.headers['content-length']).to.equal( + expectedLength.toString() + ) + }) + + it('should be able get the file back', async function() { + const res = await rp.get(fileUrl) + expect(res.body).to.equal(constantFileContent) + }) + + it('should be able to get back the first 9 bytes of the file', async function() { + const options = { + uri: fileUrl, + headers: { + Range: 'bytes=0-8' + } + } + const res = await rp.get(options) + expect(res.body).to.equal('hello wor') + }) + + it('should be able to get back bytes 4 through 10 of the file', async function() { + const options = { + uri: fileUrl, + headers: { + Range: 'bytes=4-10' + } + } + const res = await rp.get(options) + expect(res.body).to.equal('o world') + }) + + it('should be able to delete the file', async function() { + const response = await rp.del(fileUrl) + expect(response.statusCode).to.equal(204) + await expect( + rp.get(fileUrl) + ).to.eventually.be.rejected.and.have.property('statusCode', 404) + }) + + it('should be able to copy files', async function() { + const newProjectID = 'acceptance_tests_copyied_project' + const newFileId = Math.random() + const newFileUrl = `${filestoreUrl}/project/${newProjectID}/file/${newFileId}` + const opts = { + method: 'put', + uri: newFileUrl, + json: { + source: { + project_id: 'acceptance_tests', + file_id: fileId + } + } + } + let response = await rp(opts) + expect(response.statusCode).to.equal(200) + response = await rp.del(fileUrl) + expect(response.statusCode).to.equal(204) + response = await rp.get(newFileUrl) + expect(response.body).to.equal(constantFileContent) + }) + + if (backend === 'S3Persistor') { + it('should record an egress metric for the upload', async function() { + const metric = await getMetric(filestoreUrl, 's3_egress') + expect(metric - previousEgress).to.equal(constantFileContent.length) + }) + + it('should record an ingress metric when downloading the file', async function() { + await rp.get(fileUrl) + const metric = await getMetric(filestoreUrl, 's3_ingress') + expect(metric - previousIngress).to.equal( + constantFileContent.length + ) + }) + + it('should record an ingress metric for a partial download', async function() { + const options = { + uri: fileUrl, + headers: { + Range: 'bytes=0-8' + } + } + await rp.get(options) + const metric = await getMetric(filestoreUrl, 's3_ingress') + expect(metric - previousIngress).to.equal(9) + }) + } + }) + + describe('with a pdf file', function() { + let fileId, fileUrl, localFileSize + const localFileReadPath = Path.resolve( + __dirname, + '../../fixtures/test.pdf' + ) + + beforeEach(async function() { + fileId = Math.random() + fileUrl = `${filestoreUrl}/project/acceptance_tests/file/${fileId}` + const stat = await fsStat(localFileReadPath) + localFileSize = stat.size + const writeStream = request.post(fileUrl) + const endStream = fs.createWriteStream('/dev/null') + const readStream = fs.createReadStream(localFileReadPath) + await pipeline(readStream, writeStream, endStream) + }) + + it('should be able get the file back', async function() { + const response = await rp.get(fileUrl) + expect(response.body.substring(0, 8)).to.equal('%PDF-1.5') + }) + + if (backend === 'S3Persistor') { + it('should record an egress metric for the upload', async function() { + const metric = await getMetric(filestoreUrl, 's3_egress') + expect(metric - previousEgress).to.equal(localFileSize) + }) + } + + describe('getting the preview image', function() { + this.timeout(1000 * 20) + let previewFileUrl + + beforeEach(function() { + previewFileUrl = `${fileUrl}?style=preview` + }) + + it('should not time out', async function() { + const response = await rp.get(previewFileUrl) + expect(response.statusCode).to.equal(200) + }) + + it('should respond with image data', async function() { + // note: this test relies of the imagemagick conversion working + const response = await rp.get(previewFileUrl) + expect(response.body.length).to.be.greaterThan(400) + expect(response.body.substr(1, 3)).to.equal('PNG') + }) + }) + + describe('warming the cache', function() { + this.timeout(1000 * 20) + let previewFileUrl + + beforeEach(function() { + previewFileUrl = `${fileUrl}?style=preview&cacheWarm=true` + }) + + it('should not time out', async function() { + const response = await rp.get(previewFileUrl) + expect(response.statusCode).to.equal(200) + }) + + it("should respond with only an 'OK'", async function() { + // note: this test relies of the imagemagick conversion working + const response = await rp.get(previewFileUrl) + expect(response.body).to.equal('OK') + }) + }) + }) + }) + }) +}) diff --git a/services/filestore/test/acceptance/js/SendingFileTest.js b/services/filestore/test/acceptance/js/SendingFileTest.js deleted file mode 100644 index c20fa01c42..0000000000 --- a/services/filestore/test/acceptance/js/SendingFileTest.js +++ /dev/null @@ -1,326 +0,0 @@ -/* eslint-disable - handle-callback-err, - no-path-concat, - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS103: Rewrite code to no longer use __guard__ - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const { assert } = require('chai') -const sinon = require('sinon') -const chai = require('chai') -const should = chai.should() -const { expect } = chai -const modulePath = '../../../app/js/LocalFileWriter.js' -const SandboxedModule = require('sandboxed-module') -const fs = require('fs') -const request = require('request') -const settings = require('settings-sharelatex') -const FilestoreApp = require('./FilestoreApp') -const async = require('async') - -const getMetric = (filestoreUrl, metric, cb) => - request.get(`${filestoreUrl}/metrics`, function(err, res) { - expect(res.statusCode).to.equal(200) - const metricRegex = new RegExp(`^${metric}{[^}]+} ([0-9]+)$`, 'm') - return cb(parseInt(__guard__(metricRegex.exec(res.body), x => x[1]) || '0')) - }) - -describe('Filestore', function() { - before(function(done) { - this.localFileReadPath = '/tmp/filestore_acceptence_tests_file_read.txt' - this.localFileWritePath = '/tmp/filestore_acceptence_tests_file_write.txt' - - this.constantFileContent = [ - 'hello world', - `line 2 goes here ${Math.random()}`, - 'there are 3 lines in all' - ].join('\n') - - this.filestoreUrl = `http://localhost:${settings.internal.filestore.port}` - return fs.writeFile( - this.localFileReadPath, - this.constantFileContent, - function(err) { - if (err) { - return done(err) - } - return FilestoreApp.waitForS3(done) - } - ) - }) - - beforeEach(function(done) { - return FilestoreApp.ensureRunning(() => { - return async.parallel( - [ - cb => { - return fs.unlink(this.localFileWritePath, () => cb()) - }, - cb => { - return getMetric(this.filestoreUrl, 's3_egress', metric => { - this.previousEgress = metric - return cb() - }) - }, - cb => { - return getMetric(this.filestoreUrl, 's3_ingress', metric => { - this.previousIngress = metric - return cb() - }) - } - ], - done - ) - }) - }) - - it('should send a 200 for status endpoint', function(done) { - return request(`${this.filestoreUrl}/status`, function( - err, - response, - body - ) { - response.statusCode.should.equal(200) - body.indexOf('filestore').should.not.equal(-1) - body.indexOf('up').should.not.equal(-1) - return done() - }) - }) - - describe('with a file on the server', function() { - beforeEach(function(done) { - this.timeout(1000 * 10) - this.file_id = Math.random() - this.fileUrl = `${this.filestoreUrl}/project/acceptence_tests/file/${this.file_id}` - - const writeStream = request.post(this.fileUrl) - - writeStream.on('end', done) - return fs.createReadStream(this.localFileReadPath).pipe(writeStream) - }) - - it('should return 404 for a non-existant id', function(done) { - this.timeout(1000 * 20) - const options = { uri: this.fileUrl + '___this_is_clearly_wrong___' } - return request.get(options, (err, response, body) => { - response.statusCode.should.equal(404) - return done() - }) - }) - - it('should record an egress metric for the upload', function(done) { - return getMetric(this.filestoreUrl, 's3_egress', metric => { - expect(metric - this.previousEgress).to.equal( - this.constantFileContent.length - ) - return done() - }) - }) - - it('should return the file size on a HEAD request', function(done) { - const expectedLength = Buffer.byteLength(this.constantFileContent) - return request.head(this.fileUrl, (err, res) => { - expect(res.statusCode).to.equal(200) - expect(res.headers['content-length']).to.equal( - expectedLength.toString() - ) - return done() - }) - }) - - it('should be able get the file back', function(done) { - this.timeout(1000 * 10) - return request.get(this.fileUrl, (err, response, body) => { - body.should.equal(this.constantFileContent) - return done() - }) - }) - - it('should record an ingress metric when downloading the file', function(done) { - this.timeout(1000 * 10) - return request.get(this.fileUrl, () => { - return getMetric(this.filestoreUrl, 's3_ingress', metric => { - expect(metric - this.previousIngress).to.equal( - this.constantFileContent.length - ) - return done() - }) - }) - }) - - it('should be able to get back the first 9 bytes of the file', function(done) { - this.timeout(1000 * 10) - const options = { - uri: this.fileUrl, - headers: { - Range: 'bytes=0-8' - } - } - return request.get(options, (err, response, body) => { - body.should.equal('hello wor') - return done() - }) - }) - - it('should record an ingress metric for a partial download', function(done) { - this.timeout(1000 * 10) - const options = { - uri: this.fileUrl, - headers: { - Range: 'bytes=0-8' - } - } - return request.get(options, () => { - return getMetric(this.filestoreUrl, 's3_ingress', metric => { - expect(metric - this.previousIngress).to.equal(9) - return done() - }) - }) - }) - - it('should be able to get back bytes 4 through 10 of the file', function(done) { - this.timeout(1000 * 10) - const options = { - uri: this.fileUrl, - headers: { - Range: 'bytes=4-10' - } - } - return request.get(options, (err, response, body) => { - body.should.equal('o world') - return done() - }) - }) - - it('should be able to delete the file', function(done) { - this.timeout(1000 * 20) - return request.del(this.fileUrl, (err, response, body) => { - response.statusCode.should.equal(204) - return request.get(this.fileUrl, (err, response, body) => { - response.statusCode.should.equal(404) - return done() - }) - }) - }) - - return it('should be able to copy files', function(done) { - this.timeout(1000 * 20) - - const newProjectID = 'acceptence_tests_copyied_project' - const newFileId = Math.random() - const newFileUrl = `${this.filestoreUrl}/project/${newProjectID}/file/${newFileId}` - const opts = { - method: 'put', - uri: newFileUrl, - json: { - source: { - project_id: 'acceptence_tests', - file_id: this.file_id - } - } - } - return request(opts, (err, response, body) => { - response.statusCode.should.equal(200) - return request.del(this.fileUrl, (err, response, body) => { - response.statusCode.should.equal(204) - return request.get(newFileUrl, (err, response, body) => { - body.should.equal(this.constantFileContent) - return done() - }) - }) - }) - }) - }) - - return describe('with a pdf file', function() { - beforeEach(function(done) { - this.timeout(1000 * 10) - this.file_id = Math.random() - this.fileUrl = `${this.filestoreUrl}/project/acceptence_tests/file/${this.file_id}` - this.localFileReadPath = __dirname + '/../../fixtures/test.pdf' - return fs.stat(this.localFileReadPath, (err, stat) => { - this.localFileSize = stat.size - const writeStream = request.post(this.fileUrl) - - writeStream.on('end', done) - return fs.createReadStream(this.localFileReadPath).pipe(writeStream) - }) - }) - - it('should record an egress metric for the upload', function(done) { - return getMetric(this.filestoreUrl, 's3_egress', metric => { - expect(metric - this.previousEgress).to.equal(this.localFileSize) - return done() - }) - }) - - it('should be able get the file back', function(done) { - this.timeout(1000 * 10) - return request.get(this.fileUrl, (err, response, body) => { - expect(body.substring(0, 8)).to.equal('%PDF-1.5') - return done() - }) - }) - - describe('getting the preview image', function() { - beforeEach(function() { - return (this.previewFileUrl = `${this.fileUrl}?style=preview`) - }) - - it('should not time out', function(done) { - this.timeout(1000 * 20) - return request.get(this.previewFileUrl, (err, response, body) => { - expect(response).to.not.equal(null) - return done() - }) - }) - - return it('should respond with image data', function(done) { - // note: this test relies of the imagemagick conversion working - this.timeout(1000 * 20) - return request.get(this.previewFileUrl, (err, response, body) => { - expect(response.statusCode).to.equal(200) - expect(body.length).to.be.greaterThan(400) - return done() - }) - }) - }) - - return describe('warming the cache', function() { - beforeEach(function() { - return (this.fileUrl = this.fileUrl + '?style=preview&cacheWarm=true') - }) - - it('should not time out', function(done) { - this.timeout(1000 * 20) - return request.get(this.fileUrl, (err, response, body) => { - expect(response).to.not.equal(null) - return done() - }) - }) - - return it("should respond with only an 'OK'", function(done) { - // note: this test relies of the imagemagick conversion working - this.timeout(1000 * 20) - return request.get(this.fileUrl, (err, response, body) => { - expect(response.statusCode).to.equal(200) - body.should.equal('OK') - return done() - }) - }) - }) - }) -}) - -function __guard__(value, transform) { - return typeof value !== 'undefined' && value !== null - ? transform(value) - : undefined -} From 4315824d3c06160a7a9c9b84b530bf487cc08f6b Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Thu, 2 Jan 2020 09:53:07 +0000 Subject: [PATCH 46/73] Minor cleanup of FilestoreApp acceptance-test helper --- .../test/acceptance/js/FilestoreApp.js | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/services/filestore/test/acceptance/js/FilestoreApp.js b/services/filestore/test/acceptance/js/FilestoreApp.js index 4035262cbc..718d53bcf8 100644 --- a/services/filestore/test/acceptance/js/FilestoreApp.js +++ b/services/filestore/test/acceptance/js/FilestoreApp.js @@ -2,15 +2,18 @@ const logger = require('logger-sharelatex') const Settings = require('settings-sharelatex') const fs = require('fs') const Path = require('path') -const request = require('request') const { promisify } = require('util') const disrequire = require('disrequire') +const rp = require('request-promise-native').defaults({ + resolveWithFullResponse: true +}) const S3_TRIES = 30 logger.logger.level('info') const fsReaddir = promisify(fs.readdir) +const sleep = promisify(setTimeout) class FilestoreApp { constructor() { @@ -57,15 +60,15 @@ class FilestoreApp { async waitForInit() { while (this.initing) { - await new Promise(resolve => setTimeout(resolve, 1000)) + await sleep(1000) } } async stop() { - if (this.server) { - await new Promise(resolve => { - this.server.close(resolve) - }) + const closeServer = promisify(this.server.close).bind(this.server) + try { + await closeServer() + } finally { delete this.server } } @@ -80,17 +83,16 @@ class FilestoreApp { while (tries < S3_TRIES && !s3Available) { try { - const response = await promisify(request.get)( - `${Settings.filestore.s3.endpoint}/` - ) + const response = await rp.get(`${Settings.filestore.s3.endpoint}/`) if ([200, 404].includes(response.statusCode)) { s3Available = true } } catch (err) { + // swallow errors, as we may experience them until fake-s3 is running } finally { tries++ if (!s3Available) { - await new Promise(resolve => setTimeout(resolve, 1000)) + await sleep(1000) } } } From ce90292394a0c83cb3e085a4ebd10e578f32bf16 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Thu, 2 Jan 2020 11:29:28 +0000 Subject: [PATCH 47/73] Decaf cleanup for FSPersistorManager --- .../filestore/app/js/FSPersistorManager.js | 345 +++++----- .../test/unit/js/FSPersistorManagerTests.js | 631 ++++++------------ 2 files changed, 372 insertions(+), 604 deletions(-) diff --git a/services/filestore/app/js/FSPersistorManager.js b/services/filestore/app/js/FSPersistorManager.js index d11d839df7..ea793cfc64 100644 --- a/services/filestore/app/js/FSPersistorManager.js +++ b/services/filestore/app/js/FSPersistorManager.js @@ -1,206 +1,169 @@ -/* eslint-disable - handle-callback-err, - no-unreachable, - node/no-deprecated-api, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const logger = require('logger-sharelatex') const fs = require('fs') +const logger = require('logger-sharelatex') const path = require('path') -const LocalFileWriter = require('./LocalFileWriter') -const Errors = require('./Errors') const rimraf = require('rimraf') -const _ = require('underscore') +const Stream = require('stream') +const { promisify, callbackify } = require('util') + +const LocalFileWriter = require('./LocalFileWriter').promises +const { NotFoundError, ReadError } = require('./Errors') + +const pipeline = promisify(Stream.pipeline) +const fsUnlink = promisify(fs.unlink) +const fsOpen = promisify(fs.open) +const fsStat = promisify(fs.stat) +const fsReaddir = promisify(fs.readdir) +const rmrf = promisify(rimraf) const filterName = key => key.replace(/\//g, '_') -module.exports = { - sendFile(location, target, source, callback) { - if (callback == null) { - callback = function(err) {} - } - const filteredTarget = filterName(target) - logger.log({ location, target: filteredTarget, source }, 'sending file') - const done = _.once(function(err) { - if (err != null) { - logger.err( - { err, location, target: filteredTarget, source }, - 'Error on put of file' - ) - } - return callback(err) - }) - // actually copy the file (instead of moving it) to maintain consistent behaviour - // between the different implementations - const sourceStream = fs.createReadStream(source) - sourceStream.on('error', done) - const targetStream = fs.createWriteStream(`${location}/${filteredTarget}`) - targetStream.on('error', done) - targetStream.on('finish', () => done()) - return sourceStream.pipe(targetStream) - }, +async function sendFile(location, target, source) { + const filteredTarget = filterName(target) + logger.log({ location, target: filteredTarget, source }, 'sending file') - sendStream(location, target, sourceStream, callback) { - if (callback == null) { - callback = function(err) {} - } - logger.log({ location, target }, 'sending file stream') - sourceStream.on('error', err => - logger.err({ location, target, err: err('error on stream to send') }) - ) - return LocalFileWriter.writeStream(sourceStream, null, (err, fsPath) => { - if (err != null) { - logger.err( - { location, target, fsPath, err }, - 'something went wrong writing stream to disk' - ) - return callback(err) - } - return this.sendFile(location, target, fsPath, ( - err // delete the temporary file created above and return the original error - ) => LocalFileWriter.deleteFile(fsPath, () => callback(err))) - }) - }, + // actually copy the file (instead of moving it) to maintain consistent behaviour + // between the different implementations + const sourceStream = fs.createReadStream(source) + const targetStream = fs.createWriteStream(`${location}/${filteredTarget}`) + await pipeline(sourceStream, targetStream) +} - // opts may be {start: Number, end: Number} - getFileStream(location, name, opts, callback) { - if (callback == null) { - callback = function(err, res) {} - } - const filteredName = filterName(name) - logger.log({ location, filteredName }, 'getting file') - return fs.open(`${location}/${filteredName}`, 'r', function(err, fd) { - if (err != null) { - logger.err( - { err, location, filteredName: name }, - 'Error reading from file' - ) - if (err.code === 'ENOENT') { - return callback(new Errors.NotFoundError(err.message), null) - } else { - return callback(err, null) - } - } - opts.fd = fd - const sourceStream = fs.createReadStream(null, opts) - return callback(null, sourceStream) - }) - }, +async function sendStream(location, target, sourceStream) { + logger.log({ location, target }, 'sending file stream') - getFileSize(location, filename, callback) { - const fullPath = path.join(location, filterName(filename)) - return fs.stat(fullPath, function(err, stats) { - if (err != null) { - if (err.code === 'ENOENT') { - logger.log({ location, filename }, 'file not found') - callback(new Errors.NotFoundError(err.message)) - } else { - logger.err({ err, location, filename }, 'failed to stat file') - callback(err) + let fsPath + try { + fsPath = await LocalFileWriter.writeStream(sourceStream) + await sendFile(location, target, fsPath) + } finally { + await LocalFileWriter.deleteFile(fsPath) + } +} + +// opts may be {start: Number, end: Number} +async function getFileStream(location, name, opts) { + const filteredName = filterName(name) + logger.log({ location, filteredName }, 'getting file') + + try { + opts.fd = await fsOpen(`${location}/${filteredName}`, 'r') + } catch (err) { + logger.err({ err, location, filteredName: name }, 'Error reading from file') + + if (err.code === 'ENOENT') { + throw new NotFoundError({ + message: 'file not found', + info: { + location, + filteredName + } + }).withCause(err) + } + throw new ReadError('failed to open file for streaming').withCause(err) + } + + return fs.createReadStream(null, opts) +} + +async function getFileSize(location, filename) { + const fullPath = path.join(location, filterName(filename)) + + try { + const stat = await fsStat(fullPath) + return stat.size + } catch (err) { + logger.err({ err, location, filename }, 'failed to stat file') + + if (err.code === 'ENOENT') { + throw new NotFoundError({ + message: 'file not found', + info: { + location, + fullPath } - return - } - return callback(null, stats.size) - }) - }, - - copyFile(location, fromName, toName, callback) { - if (callback == null) { - callback = function(err) {} + }).withCause(err) } - const filteredFromName = filterName(fromName) - const filteredToName = filterName(toName) - logger.log( - { location, fromName: filteredFromName, toName: filteredToName }, - 'copying file' - ) - const sourceStream = fs.createReadStream(`${location}/${filteredFromName}`) - sourceStream.on('error', function(err) { - logger.err( - { err, location, key: filteredFromName }, - 'Error reading from file' - ) - return callback(err) - }) - const targetStream = fs.createWriteStream(`${location}/${filteredToName}`) - targetStream.on('error', function(err) { - logger.err( - { err, location, key: filteredToName }, - 'Error writing to file' - ) - return callback(err) - }) - targetStream.on('finish', () => callback(null)) - return sourceStream.pipe(targetStream) - }, - - deleteFile(location, name, callback) { - const filteredName = filterName(name) - logger.log({ location, filteredName }, 'delete file') - return fs.unlink(`${location}/${filteredName}`, function(err) { - if (err != null) { - logger.err({ err, location, filteredName }, 'Error on delete.') - return callback(err) - } else { - return callback() - } - }) - }, - - deleteDirectory(location, name, callback) { - if (callback == null) { - callback = function(err) {} + throw new ReadError('failed to stat file').withCause(err) + } +} + +async function copyFile(location, fromName, toName) { + const filteredFromName = filterName(fromName) + const filteredToName = filterName(toName) + logger.log({ location, filteredFromName, filteredToName }, 'copying file') + + const sourceStream = fs.createReadStream(`${location}/${filteredFromName}`) + const targetStream = fs.createWriteStream(`${location}/${filteredToName}`) + await pipeline(sourceStream, targetStream) +} + +async function deleteFile(location, name) { + const filteredName = filterName(name) + logger.log({ location, filteredName }, 'delete file') + await fsUnlink(`${location}/${filteredName}`) +} + +async function deleteDirectory(location, name) { + const filteredName = filterName(name.replace(/\/$/, '')) + + logger.log({ location, filteredName }, 'deleting directory') + + await rmrf(`${location}/${filteredName}`) +} + +async function checkIfFileExists(location, name) { + const filteredName = filterName(name) + try { + const stat = await fsStat(`${location}/${filteredName}`) + return !!stat + } catch (err) { + if (err.code === 'ENOENT') { + return false } - const filteredName = filterName(name.replace(/\/$/, '')) - return rimraf(`${location}/${filteredName}`, function(err) { - if (err != null) { - logger.err({ err, location, filteredName }, 'Error on rimraf rmdir.') - return callback(err) - } else { - return callback() - } - }) - }, - - checkIfFileExists(location, name, callback) { - if (callback == null) { - callback = function(err, exists) {} + throw new ReadError('failed to stat file').withCause(err) + } +} + +// note, does not recurse into subdirectories +async function directorySize(location, name) { + const filteredName = filterName(name.replace(/\/$/, '')) + let size = 0 + + try { + const files = await fsReaddir(`${location}/${filteredName}`) + for (const file of files) { + const stat = await fsStat(`${location}/${filteredName}/${file}`) + size += stat.size } - const filteredName = filterName(name) - logger.log({ location, filteredName }, 'checking if file exists') - return fs.exists(`${location}/${filteredName}`, function(exists) { - logger.log({ location, filteredName, exists }, 'checked if file exists') - return callback(null, exists) - }) - }, - - directorySize(location, name, callback) { - const filteredName = filterName(name.replace(/\/$/, '')) - logger.log({ location, filteredName }, 'get project size in file system') - return fs.readdir(`${location}/${filteredName}`, function(err, files) { - if (err != null) { - logger.err( - { err, location, filteredName }, - 'something went wrong listing prefix in aws' - ) - return callback(err) - } - let totalSize = 0 - _.each(files, function(entry) { - const fd = fs.openSync(`${location}/${filteredName}/${entry}`, 'r') - const fileStats = fs.fstatSync(fd) - totalSize += fileStats.size - return fs.closeSync(fd) - }) - logger.log({ totalSize }, 'total size', { files }) - return callback(null, totalSize) - }) + } catch (err) { + throw new ReadError({ + message: 'failed to get directory size', + info: { location, name } + }).withCause(err) + } + + return size +} + +module.exports = { + sendFile: callbackify(sendFile), + sendStream: callbackify(sendStream), + getFileStream: callbackify(getFileStream), + getFileSize: callbackify(getFileSize), + copyFile: callbackify(copyFile), + deleteFile: callbackify(deleteFile), + deleteDirectory: callbackify(deleteDirectory), + checkIfFileExists: callbackify(checkIfFileExists), + directorySize: callbackify(directorySize), + promises: { + sendFile, + sendStream, + getFileStream, + getFileSize, + copyFile, + deleteFile, + deleteDirectory, + checkIfFileExists, + directorySize } } diff --git a/services/filestore/test/unit/js/FSPersistorManagerTests.js b/services/filestore/test/unit/js/FSPersistorManagerTests.js index 9e9018c17b..d399a87cee 100644 --- a/services/filestore/test/unit/js/FSPersistorManagerTests.js +++ b/services/filestore/test/unit/js/FSPersistorManagerTests.js @@ -1,502 +1,307 @@ -/* eslint-disable - handle-callback-err, - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const { assert } = require('chai') const sinon = require('sinon') const chai = require('chai') -const { should } = chai const { expect } = chai -const modulePath = '../../../app/js/FSPersistorManager.js' const SandboxedModule = require('sandboxed-module') -const fs = require('fs') -const response = require('response') +const Errors = require('../../../app/js/Errors') + +chai.use(require('sinon-chai')) +chai.use(require('chai-as-promised')) + +const modulePath = '../../../app/js/FSPersistorManager.js' describe('FSPersistorManagerTests', function() { + const stat = { size: 4 } + const fd = 1234 + const readStream = 'readStream' + const writeStream = 'writeStream' + const remoteStream = 'remoteStream' + const tempFile = '/tmp/potato.txt' + const location = '/foo' + const error = new Error('guru meditation error') + + const files = ['wombat.txt', 'potato.tex'] + let fs, rimraf, stream, LocalFileWriter, FSPersistorManager + beforeEach(function() { - this.Fs = { - rename: sinon.stub(), - createReadStream: sinon.stub(), - createWriteStream: sinon.stub(), - unlink: sinon.stub(), - rmdir: sinon.stub(), - exists: sinon.stub(), - readdir: sinon.stub(), - open: sinon.stub(), - openSync: sinon.stub(), - fstatSync: sinon.stub(), - closeSync: sinon.stub(), - stat: sinon.stub() + fs = { + createReadStream: sinon.stub().returns(readStream), + createWriteStream: sinon.stub().returns(writeStream), + unlink: sinon.stub().yields(), + open: sinon.stub().yields(null, fd), + readdir: sinon.stub().yields(null, files), + stat: sinon.stub().yields(null, stat) } - this.Rimraf = sinon.stub() - this.LocalFileWriter = { - writeStream: sinon.stub(), - deleteFile: sinon.stub() + rimraf = sinon.stub().yields() + stream = { pipeline: sinon.stub().yields() } + LocalFileWriter = { + promises: { + writeStream: sinon.stub().resolves(tempFile), + deleteFile: sinon.stub().resolves() + } } - this.requires = { - './LocalFileWriter': this.LocalFileWriter, - fs: this.Fs, - 'logger-sharelatex': { - log() {}, - err() {} + FSPersistorManager = SandboxedModule.require(modulePath, { + requires: { + './LocalFileWriter': LocalFileWriter, + fs: fs, + 'logger-sharelatex': { + log() {}, + err() {} + }, + rimraf: rimraf, + stream: stream, + './Errors': Errors }, - response: response, - rimraf: this.Rimraf, - './Errors': (this.Errors = { NotFoundError: sinon.stub() }) - } - this.location = '/tmp' - this.name1 = '530f2407e7ef165704000007/530f838b46d9a9e859000008' - this.name1Filtered = '530f2407e7ef165704000007_530f838b46d9a9e859000008' - this.name2 = 'second_file' - this.error = 'error_message' - return (this.FSPersistorManager = SandboxedModule.require(modulePath, { - requires: this.requires - })) + globals: { console } + }) }) describe('sendFile', function() { - beforeEach(function() { - return (this.Fs.createReadStream = sinon.stub().returns({ - on() {}, - pipe() {} - })) + it('should copy the file', async function() { + await FSPersistorManager.promises.sendFile(location, files[0], files[1]) + expect(fs.createReadStream).to.have.been.calledWith(files[1]) + expect(fs.createWriteStream).to.have.been.calledWith( + `${location}/${files[0]}` + ) + expect(stream.pipeline).to.have.been.calledWith(readStream, writeStream) }) - it('should copy the file', function(done) { - this.Fs.createWriteStream = sinon.stub().returns({ - on(event, handler) { - if (event === 'finish') { - return process.nextTick(handler) - } - } - }) - return this.FSPersistorManager.sendFile( - this.location, - this.name1, - this.name2, - err => { - this.Fs.createReadStream.calledWith(this.name2).should.equal(true) - this.Fs.createWriteStream - .calledWith(`${this.location}/${this.name1Filtered}`) - .should.equal(true) - return done() - } - ) - }) - - return it('should return an error if the file cannot be stored', function(done) { - this.Fs.createWriteStream = sinon.stub().returns({ - on: (event, handler) => { - if (event === 'error') { - return process.nextTick(() => { - return handler(this.error) - }) - } - } - }) - return this.FSPersistorManager.sendFile( - this.location, - this.name1, - this.name2, - err => { - this.Fs.createReadStream.calledWith(this.name2).should.equal(true) - this.Fs.createWriteStream - .calledWith(`${this.location}/${this.name1Filtered}`) - .should.equal(true) - err.should.equal(this.error) - return done() - } - ) + it('should return an error if the file cannot be stored', async function() { + stream.pipeline.yields(error) + await expect( + FSPersistorManager.promises.sendFile(location, files[0], files[1]) + ).to.eventually.be.rejectedWith(error) }) }) describe('sendStream', function() { - beforeEach(function() { - this.FSPersistorManager.sendFile = sinon.stub().callsArgWith(3) - this.LocalFileWriter.writeStream.callsArgWith(2, null, this.name1) - this.LocalFileWriter.deleteFile.callsArg(1) - return (this.SourceStream = { on() {} }) - }) - - it('should sent stream to LocalFileWriter', function(done) { - return this.FSPersistorManager.sendStream( - this.location, - this.name1, - this.SourceStream, - () => { - this.LocalFileWriter.writeStream - .calledWith(this.SourceStream) - .should.equal(true) - return done() - } + it('should send the stream to LocalFileWriter', async function() { + await FSPersistorManager.promises.sendStream( + location, + files[0], + remoteStream + ) + expect(LocalFileWriter.promises.writeStream).to.have.been.calledWith( + remoteStream ) }) - it('should return the error from LocalFileWriter', function(done) { - this.LocalFileWriter.writeStream.callsArgWith(2, this.error) - return this.FSPersistorManager.sendStream( - this.location, - this.name1, - this.SourceStream, - err => { - err.should.equal(this.error) - return done() - } + it('should delete the temporary file', async function() { + await FSPersistorManager.promises.sendStream( + location, + files[0], + remoteStream + ) + expect(LocalFileWriter.promises.deleteFile).to.have.been.calledWith( + tempFile ) }) - return it('should send the file to the filestore', function(done) { - this.LocalFileWriter.writeStream.callsArgWith(2) - return this.FSPersistorManager.sendStream( - this.location, - this.name1, - this.SourceStream, - err => { - this.FSPersistorManager.sendFile.called.should.equal(true) - return done() - } + it('should return the error from LocalFileWriter', async function() { + LocalFileWriter.promises.writeStream.rejects(error) + await expect( + FSPersistorManager.promises.sendStream(location, files[0], remoteStream) + ).to.eventually.be.rejectedWith(error) + }) + + it('should send the temporary file to the filestore', async function() { + await FSPersistorManager.promises.sendStream( + location, + files[0], + remoteStream ) + expect(fs.createReadStream).to.have.been.calledWith(tempFile) }) }) describe('getFileStream', function() { - beforeEach(function() { - return (this.opts = {}) + const filename = 'wombat/potato' + const filteredFilename = 'wombat_potato' + + it('should use correct file location', async function() { + await FSPersistorManager.promises.getFileStream(location, filename, {}) + expect(fs.open).to.have.been.calledWith(`${location}/${filteredFilename}`) }) - it('should use correct file location', function(done) { - this.FSPersistorManager.getFileStream( - this.location, - this.name1, - this.opts, - (err, res) => {} + it('should pass the options to createReadStream', async function() { + await FSPersistorManager.promises.getFileStream(location, filename, { + start: 0, + end: 8 + }) + expect(fs.createReadStream).to.have.been.calledWith(null, { + start: 0, + end: 8, + fd + }) + }) + + it('should give a NotFoundError if the file does not exist', async function() { + const err = new Error() + err.code = 'ENOENT' + fs.open.yields(err) + + await expect( + FSPersistorManager.promises.getFileStream(location, filename, {}) ) - this.Fs.open - .calledWith(`${this.location}/${this.name1Filtered}`) - .should.equal(true) - return done() + .to.eventually.be.rejectedWith('file not found') + .and.be.an.instanceOf(Errors.NotFoundError) }) - describe('with start and end options', function() { - beforeEach(function() { - this.fd = 2019 - this.opts_in = { start: 0, end: 8 } - this.opts = { start: 0, end: 8, fd: this.fd } - return this.Fs.open.callsArgWith(2, null, this.fd) - }) - - return it('should pass the options to createReadStream', function(done) { - this.FSPersistorManager.getFileStream( - this.location, - this.name1, - this.opts_in, - (err, res) => {} - ) - this.Fs.createReadStream.calledWith(null, this.opts).should.equal(true) - return done() - }) - }) - - return describe('error conditions', function() { - describe('when the file does not exist', function() { - beforeEach(function() { - this.fakeCode = 'ENOENT' - const err = new Error() - err.code = this.fakeCode - return this.Fs.open.callsArgWith(2, err, null) - }) - - return it('should give a NotFoundError', function(done) { - return this.FSPersistorManager.getFileStream( - this.location, - this.name1, - this.opts, - (err, res) => { - expect(res).to.equal(null) - expect(err).to.not.equal(null) - expect(err instanceof this.Errors.NotFoundError).to.equal(true) - return done() - } - ) - }) - }) - - return describe('when some other error happens', function() { - beforeEach(function() { - this.fakeCode = 'SOMETHINGHORRIBLE' - const err = new Error() - err.code = this.fakeCode - return this.Fs.open.callsArgWith(2, err, null) - }) - - return it('should give an Error', function(done) { - return this.FSPersistorManager.getFileStream( - this.location, - this.name1, - this.opts, - (err, res) => { - expect(res).to.equal(null) - expect(err).to.not.equal(null) - expect(err instanceof Error).to.equal(true) - return done() - } - ) - }) - }) + it('should wrap any other error', async function() { + fs.open.yields(error) + await expect( + FSPersistorManager.promises.getFileStream(location, filename, {}) + ) + .to.eventually.be.rejectedWith('failed to open file for streaming') + .and.be.an.instanceOf(Errors.ReadError) + .and.have.property('cause', error) }) }) describe('getFileSize', function() { - it('should return the file size', function(done) { - const expectedFileSize = 75382 - this.Fs.stat.yields(new Error('fs.stat got unexpected arguments')) - this.Fs.stat - .withArgs(`${this.location}/${this.name1Filtered}`) - .yields(null, { size: expectedFileSize }) + const filename = 'wombat/potato' + const badFilename = 'neenaw.tex' + const filteredFilename = 'wombat_potato' + const size = 65536 + const noentError = new Error('not found') + noentError.code = 'ENOENT' - return this.FSPersistorManager.getFileSize( - this.location, - this.name1, - (err, fileSize) => { - if (err != null) { - return done(err) - } - expect(fileSize).to.equal(expectedFileSize) - return done() - } - ) + beforeEach(function() { + fs.stat + .yields(error) + .withArgs(`${location}/${filteredFilename}`) + .yields(null, { size }) + .withArgs(`${location}/${badFilename}`) + .yields(noentError) }) - it('should throw a NotFoundError if the file does not exist', function(done) { - const error = new Error() - error.code = 'ENOENT' - this.Fs.stat.yields(error) - - return this.FSPersistorManager.getFileSize( - this.location, - this.name1, - (err, fileSize) => { - expect(err).to.be.instanceof(this.Errors.NotFoundError) - return done() - } - ) + it('should return the file size', async function() { + expect( + await FSPersistorManager.promises.getFileSize(location, filename) + ).to.equal(size) }) - return it('should rethrow any other error', function(done) { - const error = new Error() - this.Fs.stat.yields(error) + it('should throw a NotFoundError if the file does not exist', async function() { + await expect( + FSPersistorManager.promises.getFileSize(location, badFilename) + ).to.eventually.be.rejected.and.be.an.instanceOf(Errors.NotFoundError) + }) - return this.FSPersistorManager.getFileSize( - this.location, - this.name1, - (err, fileSize) => { - expect(err).to.equal(error) - return done() - } - ) + it('should wrap any other error', async function() { + await expect(FSPersistorManager.promises.getFileSize(location, 'raccoon')) + .to.eventually.be.rejected.and.be.an.instanceOf(Errors.ReadError) + .and.have.property('cause', error) }) }) describe('copyFile', function() { - beforeEach(function() { - this.ReadStream = { - on() {}, - pipe: sinon.stub() - } - this.WriteStream = { on() {} } - this.Fs.createReadStream.returns(this.ReadStream) - return this.Fs.createWriteStream.returns(this.WriteStream) + it('Should open the source for reading', async function() { + await FSPersistorManager.promises.copyFile(location, files[0], files[1]) + expect(fs.createReadStream).to.have.been.calledWith( + `${location}/${files[0]}` + ) }) - it('Should open the source for reading', function(done) { - this.FSPersistorManager.copyFile( - this.location, - this.name1, - this.name2, - function() {} + it('Should open the target for writing', async function() { + await FSPersistorManager.promises.copyFile(location, files[0], files[1]) + expect(fs.createWriteStream).to.have.been.calledWith( + `${location}/${files[1]}` ) - this.Fs.createReadStream - .calledWith(`${this.location}/${this.name1Filtered}`) - .should.equal(true) - return done() }) - it('Should open the target for writing', function(done) { - this.FSPersistorManager.copyFile( - this.location, - this.name1, - this.name2, - function() {} - ) - this.Fs.createWriteStream - .calledWith(`${this.location}/${this.name2}`) - .should.equal(true) - return done() - }) - - return it('Should pipe the source to the target', function(done) { - this.FSPersistorManager.copyFile( - this.location, - this.name1, - this.name2, - function() {} - ) - this.ReadStream.pipe.calledWith(this.WriteStream).should.equal(true) - return done() + it('Should pipe the source to the target', async function() { + await FSPersistorManager.promises.copyFile(location, files[0], files[1]) + expect(stream.pipeline).to.have.been.calledWith(readStream, writeStream) }) }) describe('deleteFile', function() { - beforeEach(function() { - return this.Fs.unlink.callsArgWith(1, this.error) + it('Should call unlink with correct options', async function() { + await FSPersistorManager.promises.deleteFile(location, files[0]) + expect(fs.unlink).to.have.been.calledWith(`${location}/${files[0]}`) }) - it('Should call unlink with correct options', function(done) { - return this.FSPersistorManager.deleteFile( - this.location, - this.name1, - err => { - this.Fs.unlink - .calledWith(`${this.location}/${this.name1Filtered}`) - .should.equal(true) - return done() - } - ) - }) - - return it('Should propogate the error', function(done) { - return this.FSPersistorManager.deleteFile( - this.location, - this.name1, - err => { - err.should.equal(this.error) - return done() - } - ) + it('Should propagate the error', async function() { + fs.unlink.yields(error) + await expect( + FSPersistorManager.promises.deleteFile(location, files[0]) + ).to.eventually.be.rejectedWith(error) }) }) describe('deleteDirectory', function() { - beforeEach(function() { - return this.Rimraf.callsArgWith(1, this.error) + it('Should call rmdir(rimraf) with correct options', async function() { + await FSPersistorManager.promises.deleteDirectory(location, files[0]) + expect(rimraf).to.have.been.calledWith(`${location}/${files[0]}`) }) - it('Should call rmdir(rimraf) with correct options', function(done) { - return this.FSPersistorManager.deleteDirectory( - this.location, - this.name1, - err => { - this.Rimraf.calledWith( - `${this.location}/${this.name1Filtered}` - ).should.equal(true) - return done() - } - ) - }) - - return it('Should propogate the error', function(done) { - return this.FSPersistorManager.deleteDirectory( - this.location, - this.name1, - err => { - err.should.equal(this.error) - return done() - } - ) + it('Should propagate the error', async function() { + rimraf.yields(error) + await expect( + FSPersistorManager.promises.deleteDirectory(location, files[0]) + ).to.eventually.be.rejectedWith(error) }) }) describe('checkIfFileExists', function() { + const filename = 'wombat' + const badFilename = 'potato' + const noentError = new Error('not found') + noentError.code = 'ENOENT' + beforeEach(function() { - return this.Fs.exists.callsArgWith(1, true) + fs.stat + .yields(error) + .withArgs(`${location}/${filename}`) + .yields(null, {}) + .withArgs(`${location}/${badFilename}`) + .yields(noentError) }) - it('Should call exists with correct options', function(done) { - return this.FSPersistorManager.checkIfFileExists( - this.location, - this.name1, - exists => { - this.Fs.exists - .calledWith(`${this.location}/${this.name1Filtered}`) - .should.equal(true) - return done() - } - ) + it('Should call stat with correct options', async function() { + await FSPersistorManager.promises.checkIfFileExists(location, filename) + expect(fs.stat).to.have.been.calledWith(`${location}/${filename}`) }) - // fs.exists simply returns false on any error, so... - it('should not return an error', function(done) { - return this.FSPersistorManager.checkIfFileExists( - this.location, - this.name1, - (err, exists) => { - expect(err).to.be.null - return done() - } - ) + it('Should return true for existing files', async function() { + expect( + await FSPersistorManager.promises.checkIfFileExists(location, filename) + ).to.equal(true) }) - it('Should return true for existing files', function(done) { - this.Fs.exists.callsArgWith(1, true) - return this.FSPersistorManager.checkIfFileExists( - this.location, - this.name1, - (err, exists) => { - exists.should.be.true - return done() - } - ) + it('Should return false for non-existing files', async function() { + expect( + await FSPersistorManager.promises.checkIfFileExists( + location, + badFilename + ) + ).to.equal(false) }) - return it('Should return false for non-existing files', function(done) { - this.Fs.exists.callsArgWith(1, false) - return this.FSPersistorManager.checkIfFileExists( - this.location, - this.name1, - (err, exists) => { - exists.should.be.false - return done() - } + it('should wrap the error if there is a problem', async function() { + await expect( + FSPersistorManager.promises.checkIfFileExists(location, 'llama') ) + .to.eventually.be.rejected.and.be.an.instanceOf(Errors.ReadError) + .and.have.property('cause', error) }) }) - return describe('directorySize', function() { - it('should propogate the error', function(done) { - this.Fs.readdir.callsArgWith(1, this.error) - return this.FSPersistorManager.directorySize( - this.location, - this.name1, - (err, totalsize) => { - err.should.equal(this.error) - return done() - } + describe('directorySize', function() { + it('should wrap the error', async function() { + fs.readdir.yields(error) + await expect( + FSPersistorManager.promises.directorySize(location, 'wombat') ) + .to.eventually.be.rejected.and.be.an.instanceOf(Errors.ReadError) + .and.include({ cause: error }) + .and.have.property('info') + .which.includes({ location, name: 'wombat' }) }) - return it('should sum directory files size', function(done) { - this.Fs.readdir.callsArgWith(1, null, [ - { file1: 'file1' }, - { file2: 'file2' } - ]) - this.Fs.fstatSync.returns({ size: 1024 }) - return this.FSPersistorManager.directorySize( - this.location, - this.name1, - (err, totalsize) => { - expect(totalsize).to.equal(2048) - return done() - } - ) + it('should sum directory files size', async function() { + expect( + await FSPersistorManager.promises.directorySize(location, 'wombat') + ).to.equal(stat.size * files.length) }) }) }) From 21ba083877d6f5aa961161b2a69f458c844eca72 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 6 Jan 2020 14:09:28 +0000 Subject: [PATCH 48/73] Use 'glob' to determine 'directory' size. Check for filtered directory paths in all tests. --- .../filestore/app/js/FSPersistorManager.js | 20 +++- services/filestore/npm-shrinkwrap.json | 29 ++++-- services/filestore/package.json | 1 + .../test/unit/js/FSPersistorManagerTests.js | 96 +++++++++++-------- 4 files changed, 96 insertions(+), 50 deletions(-) diff --git a/services/filestore/app/js/FSPersistorManager.js b/services/filestore/app/js/FSPersistorManager.js index ea793cfc64..8c3ad9ae02 100644 --- a/services/filestore/app/js/FSPersistorManager.js +++ b/services/filestore/app/js/FSPersistorManager.js @@ -1,4 +1,5 @@ const fs = require('fs') +const glob = require('glob') const logger = require('logger-sharelatex') const path = require('path') const rimraf = require('rimraf') @@ -12,7 +13,7 @@ const pipeline = promisify(Stream.pipeline) const fsUnlink = promisify(fs.unlink) const fsOpen = promisify(fs.open) const fsStat = promisify(fs.stat) -const fsReaddir = promisify(fs.readdir) +const fsGlob = promisify(glob) const rmrf = promisify(rimraf) const filterName = key => key.replace(/\//g, '_') @@ -124,16 +125,25 @@ async function checkIfFileExists(location, name) { } } -// note, does not recurse into subdirectories +// note, does not recurse into subdirectories, as we use a flattened directory structure async function directorySize(location, name) { const filteredName = filterName(name.replace(/\/$/, '')) let size = 0 try { - const files = await fsReaddir(`${location}/${filteredName}`) + const files = await fsGlob(`${location}/${filteredName}_*`) for (const file of files) { - const stat = await fsStat(`${location}/${filteredName}/${file}`) - size += stat.size + try { + const stat = await fsStat(file) + if (stat.isFile()) { + size += stat.size + } + } catch (err) { + // ignore files that may have just been deleted + if (err.code !== 'ENOENT') { + throw err + } + } } } catch (err) { throw new ReadError({ diff --git a/services/filestore/npm-shrinkwrap.json b/services/filestore/npm-shrinkwrap.json index fa498f3f1b..3ed1400a61 100644 --- a/services/filestore/npm-shrinkwrap.json +++ b/services/filestore/npm-shrinkwrap.json @@ -473,7 +473,7 @@ "@sinonjs/text-encoding": { "version": "0.7.1", "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.1.tgz", - "integrity": "sha1-jaXGUwkVZT86Hzj9XxAdjD+AecU=", + "integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==", "dev": true }, "@types/caseless": { @@ -2189,14 +2189,14 @@ "integrity": "sha1-uKLHAUu1zUFTTpg7XKFgo3RwhGk=" }, "glob": { - "version": "6.0.4", - "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", - "integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=", - "optional": true, + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", "requires": { + "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", - "minimatch": "2 || 3", + "minimatch": "^3.0.4", "once": "^1.3.0", "path-is-absolute": "^1.0.0" } @@ -2671,7 +2671,7 @@ "just-extend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.0.2.tgz", - "integrity": "sha1-8/R/ffyg+YnFVBCn68iFSwcQivw=", + "integrity": "sha512-FrLwOgm+iXrPV+5zDU6Jqu4gCRXbWEQg2O3SKONsWE4w7AXFRkryS53bpWdaL9cNol+AmR3AEYz6kn+o0fCPnw==", "dev": true }, "jwa": { @@ -3259,6 +3259,21 @@ "optional": true, "requires": { "glob": "^6.0.1" + }, + "dependencies": { + "glob": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", + "integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=", + "optional": true, + "requires": { + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "2 || 3", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + } } } } diff --git a/services/filestore/package.json b/services/filestore/package.json index 9515c1850c..5d7c3e3ec1 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -26,6 +26,7 @@ "body-parser": "^1.2.0", "express": "^4.2.0", "fs-extra": "^1.0.0", + "glob": "^7.1.6", "heapdump": "^0.3.2", "knox": "~0.9.1", "logger-sharelatex": "^1.7.0", diff --git a/services/filestore/test/unit/js/FSPersistorManagerTests.js b/services/filestore/test/unit/js/FSPersistorManagerTests.js index d399a87cee..cb177989a5 100644 --- a/services/filestore/test/unit/js/FSPersistorManagerTests.js +++ b/services/filestore/test/unit/js/FSPersistorManagerTests.js @@ -10,7 +10,7 @@ chai.use(require('chai-as-promised')) const modulePath = '../../../app/js/FSPersistorManager.js' describe('FSPersistorManagerTests', function() { - const stat = { size: 4 } + const stat = { size: 4, isFile: sinon.stub().returns(true) } const fd = 1234 const readStream = 'readStream' const writeStream = 'writeStream' @@ -19,8 +19,9 @@ describe('FSPersistorManagerTests', function() { const location = '/foo' const error = new Error('guru meditation error') - const files = ['wombat.txt', 'potato.tex'] - let fs, rimraf, stream, LocalFileWriter, FSPersistorManager + const files = ['animals/wombat.tex', 'vegetables/potato.tex'] + const filteredFilenames = ['animals_wombat.tex', 'vegetables_potato.tex'] + let fs, rimraf, stream, LocalFileWriter, FSPersistorManager, glob beforeEach(function() { fs = { @@ -28,9 +29,9 @@ describe('FSPersistorManagerTests', function() { createWriteStream: sinon.stub().returns(writeStream), unlink: sinon.stub().yields(), open: sinon.stub().yields(null, fd), - readdir: sinon.stub().yields(null, files), stat: sinon.stub().yields(null, stat) } + glob = sinon.stub().yields(null, files) rimraf = sinon.stub().yields() stream = { pipeline: sinon.stub().yields() } LocalFileWriter = { @@ -42,25 +43,31 @@ describe('FSPersistorManagerTests', function() { FSPersistorManager = SandboxedModule.require(modulePath, { requires: { './LocalFileWriter': LocalFileWriter, - fs: fs, 'logger-sharelatex': { log() {}, err() {} }, - rimraf: rimraf, - stream: stream, - './Errors': Errors + './Errors': Errors, + fs, + glob, + rimraf, + stream }, globals: { console } }) }) describe('sendFile', function() { + const localFilesystemPath = '/path/to/local/file' it('should copy the file', async function() { - await FSPersistorManager.promises.sendFile(location, files[0], files[1]) - expect(fs.createReadStream).to.have.been.calledWith(files[1]) + await FSPersistorManager.promises.sendFile( + location, + files[0], + localFilesystemPath + ) + expect(fs.createReadStream).to.have.been.calledWith(localFilesystemPath) expect(fs.createWriteStream).to.have.been.calledWith( - `${location}/${files[0]}` + `${location}/${filteredFilenames[0]}` ) expect(stream.pipeline).to.have.been.calledWith(readStream, writeStream) }) @@ -68,7 +75,11 @@ describe('FSPersistorManagerTests', function() { it('should return an error if the file cannot be stored', async function() { stream.pipeline.yields(error) await expect( - FSPersistorManager.promises.sendFile(location, files[0], files[1]) + FSPersistorManager.promises.sendFile( + location, + files[0], + localFilesystemPath + ) ).to.eventually.be.rejectedWith(error) }) }) @@ -114,16 +125,15 @@ describe('FSPersistorManagerTests', function() { }) describe('getFileStream', function() { - const filename = 'wombat/potato' - const filteredFilename = 'wombat_potato' - it('should use correct file location', async function() { - await FSPersistorManager.promises.getFileStream(location, filename, {}) - expect(fs.open).to.have.been.calledWith(`${location}/${filteredFilename}`) + await FSPersistorManager.promises.getFileStream(location, files[0], {}) + expect(fs.open).to.have.been.calledWith( + `${location}/${filteredFilenames[0]}` + ) }) it('should pass the options to createReadStream', async function() { - await FSPersistorManager.promises.getFileStream(location, filename, { + await FSPersistorManager.promises.getFileStream(location, files[0], { start: 0, end: 8 }) @@ -140,7 +150,7 @@ describe('FSPersistorManagerTests', function() { fs.open.yields(err) await expect( - FSPersistorManager.promises.getFileStream(location, filename, {}) + FSPersistorManager.promises.getFileStream(location, files[0], {}) ) .to.eventually.be.rejectedWith('file not found') .and.be.an.instanceOf(Errors.NotFoundError) @@ -149,7 +159,7 @@ describe('FSPersistorManagerTests', function() { it('should wrap any other error', async function() { fs.open.yields(error) await expect( - FSPersistorManager.promises.getFileStream(location, filename, {}) + FSPersistorManager.promises.getFileStream(location, files[0], {}) ) .to.eventually.be.rejectedWith('failed to open file for streaming') .and.be.an.instanceOf(Errors.ReadError) @@ -158,9 +168,7 @@ describe('FSPersistorManagerTests', function() { }) describe('getFileSize', function() { - const filename = 'wombat/potato' const badFilename = 'neenaw.tex' - const filteredFilename = 'wombat_potato' const size = 65536 const noentError = new Error('not found') noentError.code = 'ENOENT' @@ -168,7 +176,7 @@ describe('FSPersistorManagerTests', function() { beforeEach(function() { fs.stat .yields(error) - .withArgs(`${location}/${filteredFilename}`) + .withArgs(`${location}/${filteredFilenames[0]}`) .yields(null, { size }) .withArgs(`${location}/${badFilename}`) .yields(noentError) @@ -176,7 +184,7 @@ describe('FSPersistorManagerTests', function() { it('should return the file size', async function() { expect( - await FSPersistorManager.promises.getFileSize(location, filename) + await FSPersistorManager.promises.getFileSize(location, files[0]) ).to.equal(size) }) @@ -197,14 +205,14 @@ describe('FSPersistorManagerTests', function() { it('Should open the source for reading', async function() { await FSPersistorManager.promises.copyFile(location, files[0], files[1]) expect(fs.createReadStream).to.have.been.calledWith( - `${location}/${files[0]}` + `${location}/${filteredFilenames[0]}` ) }) it('Should open the target for writing', async function() { await FSPersistorManager.promises.copyFile(location, files[0], files[1]) expect(fs.createWriteStream).to.have.been.calledWith( - `${location}/${files[1]}` + `${location}/${filteredFilenames[1]}` ) }) @@ -217,7 +225,9 @@ describe('FSPersistorManagerTests', function() { describe('deleteFile', function() { it('Should call unlink with correct options', async function() { await FSPersistorManager.promises.deleteFile(location, files[0]) - expect(fs.unlink).to.have.been.calledWith(`${location}/${files[0]}`) + expect(fs.unlink).to.have.been.calledWith( + `${location}/${filteredFilenames[0]}` + ) }) it('Should propagate the error', async function() { @@ -231,7 +241,9 @@ describe('FSPersistorManagerTests', function() { describe('deleteDirectory', function() { it('Should call rmdir(rimraf) with correct options', async function() { await FSPersistorManager.promises.deleteDirectory(location, files[0]) - expect(rimraf).to.have.been.calledWith(`${location}/${files[0]}`) + expect(rimraf).to.have.been.calledWith( + `${location}/${filteredFilenames[0]}` + ) }) it('Should propagate the error', async function() { @@ -243,28 +255,29 @@ describe('FSPersistorManagerTests', function() { }) describe('checkIfFileExists', function() { - const filename = 'wombat' - const badFilename = 'potato' + const badFilename = 'pototo' const noentError = new Error('not found') noentError.code = 'ENOENT' beforeEach(function() { fs.stat .yields(error) - .withArgs(`${location}/${filename}`) + .withArgs(`${location}/${filteredFilenames[0]}`) .yields(null, {}) .withArgs(`${location}/${badFilename}`) .yields(noentError) }) it('Should call stat with correct options', async function() { - await FSPersistorManager.promises.checkIfFileExists(location, filename) - expect(fs.stat).to.have.been.calledWith(`${location}/${filename}`) + await FSPersistorManager.promises.checkIfFileExists(location, files[0]) + expect(fs.stat).to.have.been.calledWith( + `${location}/${filteredFilenames[0]}` + ) }) it('Should return true for existing files', async function() { expect( - await FSPersistorManager.promises.checkIfFileExists(location, filename) + await FSPersistorManager.promises.checkIfFileExists(location, files[0]) ).to.equal(true) }) @@ -288,19 +301,26 @@ describe('FSPersistorManagerTests', function() { describe('directorySize', function() { it('should wrap the error', async function() { - fs.readdir.yields(error) + glob.yields(error) await expect( - FSPersistorManager.promises.directorySize(location, 'wombat') + FSPersistorManager.promises.directorySize(location, files[0]) ) .to.eventually.be.rejected.and.be.an.instanceOf(Errors.ReadError) .and.include({ cause: error }) .and.have.property('info') - .which.includes({ location, name: 'wombat' }) + .which.includes({ location, name: files[0] }) + }) + + it('should filter the directory name', async function() { + await FSPersistorManager.promises.directorySize(location, files[0]) + expect(glob).to.have.been.calledWith( + `${location}/${filteredFilenames[0]}_*` + ) }) it('should sum directory files size', async function() { expect( - await FSPersistorManager.promises.directorySize(location, 'wombat') + await FSPersistorManager.promises.directorySize(location, files[0]) ).to.equal(stat.size * files.length) }) }) From 039bec02f729f08c0dea3eabd5da9c24d0b772ca Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 6 Jan 2020 14:14:22 +0000 Subject: [PATCH 49/73] Use directory names in acceptance tests, and add test for getting size --- .../test/acceptance/js/FilestoreTests.js | 75 +++++++++++++++++-- 1 file changed, 70 insertions(+), 5 deletions(-) diff --git a/services/filestore/test/acceptance/js/FilestoreTests.js b/services/filestore/test/acceptance/js/FilestoreTests.js index 3315569a24..9260b1bd62 100644 --- a/services/filestore/test/acceptance/js/FilestoreTests.js +++ b/services/filestore/test/acceptance/js/FilestoreTests.js @@ -56,6 +56,7 @@ if (process.env.AWS_ACCESS_KEY_ID) { describe('Filestore', function() { this.timeout(1000 * 10) const filestoreUrl = `http://localhost:${Settings.internal.filestore.port}` + const directoryName = 'directory' // redefine the test suite for every available backend Object.keys(BackendSettings).forEach(backend => { @@ -113,11 +114,11 @@ describe('Filestore', function() { beforeEach(async function() { fileId = Math.random() - fileUrl = `${filestoreUrl}/project/acceptance_tests/file/${fileId}` + fileUrl = `${filestoreUrl}/project/acceptance_tests/file/${directoryName}%2F${fileId}` const writeStream = request.post(fileUrl) const readStream = fs.createReadStream(localFileReadPath) - // consume the result to ensure the http request has been fully processed + // hack to consume the result to ensure the http request has been fully processed const resultStream = fs.createWriteStream('/dev/null') await pipeline(readStream, writeStream, resultStream) }) @@ -176,14 +177,14 @@ describe('Filestore', function() { it('should be able to copy files', async function() { const newProjectID = 'acceptance_tests_copyied_project' const newFileId = Math.random() - const newFileUrl = `${filestoreUrl}/project/${newProjectID}/file/${newFileId}` + const newFileUrl = `${filestoreUrl}/project/${newProjectID}/file/${directoryName}%2F${newFileId}` const opts = { method: 'put', uri: newFileUrl, json: { source: { project_id: 'acceptance_tests', - file_id: fileId + file_id: `${directoryName}/${fileId}` } } } @@ -223,6 +224,70 @@ describe('Filestore', function() { } }) + describe('with multiple files', function() { + let fileIds, fileUrls, project + const directoryName = 'directory' + const localFileReadPaths = [ + '/tmp/filestore_acceptance_tests_file_read_1.txt', + '/tmp/filestore_acceptance_tests_file_read_2.txt' + ] + const constantFileContents = [ + [ + 'hello world', + `line 2 goes here ${Math.random()}`, + 'there are 3 lines in all' + ].join('\n'), + [ + `for reference: ${Math.random()}`, + 'cats are the best animals', + 'wombats are a close second' + ].join('\n') + ] + + before(async function() { + return Promise.all([ + fsWriteFile(localFileReadPaths[0], constantFileContents[0]), + fsWriteFile(localFileReadPaths[1], constantFileContents[1]) + ]) + }) + + beforeEach(async function() { + project = `acceptance_tests_${Math.random()}` + fileIds = [Math.random(), Math.random()] + fileUrls = [ + `${filestoreUrl}/project/${project}/file/${directoryName}%2F${fileIds[0]}`, + `${filestoreUrl}/project/${project}/file/${directoryName}%2F${fileIds[1]}` + ] + + const writeStreams = [ + request.post(fileUrls[0]), + request.post(fileUrls[1]) + ] + const readStreams = [ + fs.createReadStream(localFileReadPaths[0]), + fs.createReadStream(localFileReadPaths[1]) + ] + // hack to consume the result to ensure the http request has been fully processed + const resultStreams = [ + fs.createWriteStream('/dev/null'), + fs.createWriteStream('/dev/null') + ] + return Promise.all([ + pipeline(readStreams[0], writeStreams[0], resultStreams[0]), + pipeline(readStreams[1], writeStreams[1], resultStreams[1]) + ]) + }) + + it('should get the directory size', async function() { + const response = await rp.get( + `${filestoreUrl}/project/${project}/size` + ) + expect(parseInt(JSON.parse(response.body)['total bytes'])).to.equal( + constantFileContents[0].length + constantFileContents[1].length + ) + }) + }) + describe('with a pdf file', function() { let fileId, fileUrl, localFileSize const localFileReadPath = Path.resolve( @@ -232,7 +297,7 @@ describe('Filestore', function() { beforeEach(async function() { fileId = Math.random() - fileUrl = `${filestoreUrl}/project/acceptance_tests/file/${fileId}` + fileUrl = `${filestoreUrl}/project/acceptance_tests/file/${directoryName}%2F${fileId}` const stat = await fsStat(localFileReadPath) localFileSize = stat.size const writeStream = request.post(fileUrl) From 25f1c2bfc407a2ed156082b3cfde4186cafbe75c Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 6 Jan 2020 14:43:23 +0000 Subject: [PATCH 50/73] Delete temporary file when error in writing to stream --- services/filestore/app/js/LocalFileWriter.js | 13 ++++- .../test/unit/js/LocalFileWriterTests.js | 48 +++++++++++++++---- 2 files changed, 52 insertions(+), 9 deletions(-) diff --git a/services/filestore/app/js/LocalFileWriter.js b/services/filestore/app/js/LocalFileWriter.js index 44f3f9433a..22957e15d1 100644 --- a/services/filestore/app/js/LocalFileWriter.js +++ b/services/filestore/app/js/LocalFileWriter.js @@ -32,6 +32,8 @@ async function writeStream(stream, key) { logger.log({ fsPath }, 'finished writing file locally') return fsPath } catch (err) { + await deleteFile(fsPath) + logger.err({ err, fsPath }, 'problem writing file locally') throw new WriteError({ message: 'problem writing file locally', @@ -45,7 +47,16 @@ async function deleteFile(fsPath) { return } logger.log({ fsPath }, 'removing local temp file') - await promisify(fs.unlink)(fsPath) + try { + await promisify(fs.unlink)(fsPath) + } catch (err) { + if (err.code !== 'ENOENT') { + throw new WriteError({ + message: 'failed to delete file', + info: { fsPath } + }).withCause(err) + } + } } function _getPath(key) { diff --git a/services/filestore/test/unit/js/LocalFileWriterTests.js b/services/filestore/test/unit/js/LocalFileWriterTests.js index 5d7008a91f..ad4d73bce6 100644 --- a/services/filestore/test/unit/js/LocalFileWriterTests.js +++ b/services/filestore/test/unit/js/LocalFileWriterTests.js @@ -49,6 +49,26 @@ describe('LocalFileWriter', function() { done() }) }) + + describe('when there is an error', function() { + const error = new Error('not enough ketchup') + beforeEach(function() { + stream.pipeline.yields(error) + }) + + it('should wrap the error', function() { + LocalFileWriter.writeStream(readStream, filename, err => { + expect(err).to.exist + expect(err.cause).to.equal(error) + }) + }) + + it('should delete the temporary file', function() { + LocalFileWriter.writeStream(readStream, filename, () => { + expect(fs.unlink).to.have.been.calledWith(fsPath) + }) + }) + }) }) describe('deleteFile', function() { @@ -60,14 +80,6 @@ describe('LocalFileWriter', function() { }) }) - it('should not do anything if called with an empty path', function(done) { - fs.unlink = sinon.stub().yields(new Error('failed to reticulate splines')) - LocalFileWriter.deleteFile(fsPath, err => { - expect(err).to.exist - done() - }) - }) - it('should not call unlink with an empty path', function(done) { LocalFileWriter.deleteFile('', err => { expect(err).not.to.exist @@ -75,5 +87,25 @@ describe('LocalFileWriter', function() { done() }) }) + + it('should not throw a error if the file does not exist', function(done) { + const error = new Error('file not found') + error.code = 'ENOENT' + fs.unlink = sinon.stub().yields(error) + LocalFileWriter.deleteFile(fsPath, err => { + expect(err).not.to.exist + done() + }) + }) + + it('should wrap the error', function(done) { + const error = new Error('failed to reticulate splines') + fs.unlink = sinon.stub().yields(error) + LocalFileWriter.deleteFile(fsPath, err => { + expect(err).to.exist + expect(err.cause).to.equal(error) + done() + }) + }) }) }) From f1b6b35c691ff0c418c23203ef6821df01dedad1 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 6 Jan 2020 15:11:35 +0000 Subject: [PATCH 51/73] Throw errors more consistently in FSPersistorManager --- .../filestore/app/js/FSPersistorManager.js | 121 ++++++++++++------ .../test/unit/js/FSPersistorManagerTests.js | 10 +- 2 files changed, 90 insertions(+), 41 deletions(-) diff --git a/services/filestore/app/js/FSPersistorManager.js b/services/filestore/app/js/FSPersistorManager.js index 8c3ad9ae02..c649dfb61b 100644 --- a/services/filestore/app/js/FSPersistorManager.js +++ b/services/filestore/app/js/FSPersistorManager.js @@ -7,7 +7,7 @@ const Stream = require('stream') const { promisify, callbackify } = require('util') const LocalFileWriter = require('./LocalFileWriter').promises -const { NotFoundError, ReadError } = require('./Errors') +const { NotFoundError, ReadError, WriteError } = require('./Errors') const pipeline = promisify(Stream.pipeline) const fsUnlink = promisify(fs.unlink) @@ -24,17 +24,26 @@ async function sendFile(location, target, source) { // actually copy the file (instead of moving it) to maintain consistent behaviour // between the different implementations - const sourceStream = fs.createReadStream(source) - const targetStream = fs.createWriteStream(`${location}/${filteredTarget}`) - await pipeline(sourceStream, targetStream) + try { + const sourceStream = fs.createReadStream(source) + const targetStream = fs.createWriteStream(`${location}/${filteredTarget}`) + await pipeline(sourceStream, targetStream) + } catch (err) { + throw _wrapError( + err, + 'failed to copy the specified file', + { location, target, source }, + WriteError + ) + } } async function sendStream(location, target, sourceStream) { logger.log({ location, target }, 'sending file stream') - let fsPath + const fsPath = await LocalFileWriter.writeStream(sourceStream) + try { - fsPath = await LocalFileWriter.writeStream(sourceStream) await sendFile(location, target, fsPath) } finally { await LocalFileWriter.deleteFile(fsPath) @@ -51,16 +60,12 @@ async function getFileStream(location, name, opts) { } catch (err) { logger.err({ err, location, filteredName: name }, 'Error reading from file') - if (err.code === 'ENOENT') { - throw new NotFoundError({ - message: 'file not found', - info: { - location, - filteredName - } - }).withCause(err) - } - throw new ReadError('failed to open file for streaming').withCause(err) + throw _wrapError( + err, + 'failed to open file for streaming', + { location, filteredName, opts }, + ReadError + ) } return fs.createReadStream(null, opts) @@ -75,16 +80,12 @@ async function getFileSize(location, filename) { } catch (err) { logger.err({ err, location, filename }, 'failed to stat file') - if (err.code === 'ENOENT') { - throw new NotFoundError({ - message: 'file not found', - info: { - location, - fullPath - } - }).withCause(err) - } - throw new ReadError('failed to stat file').withCause(err) + throw _wrapError( + err, + 'failed to stat file', + { location, filename }, + ReadError + ) } } @@ -93,15 +94,33 @@ async function copyFile(location, fromName, toName) { const filteredToName = filterName(toName) logger.log({ location, filteredFromName, filteredToName }, 'copying file') - const sourceStream = fs.createReadStream(`${location}/${filteredFromName}`) - const targetStream = fs.createWriteStream(`${location}/${filteredToName}`) - await pipeline(sourceStream, targetStream) + try { + const sourceStream = fs.createReadStream(`${location}/${filteredFromName}`) + const targetStream = fs.createWriteStream(`${location}/${filteredToName}`) + await pipeline(sourceStream, targetStream) + } catch (err) { + throw _wrapError( + err, + 'failed to copy file', + { location, filteredFromName, filteredToName }, + WriteError + ) + } } async function deleteFile(location, name) { const filteredName = filterName(name) logger.log({ location, filteredName }, 'delete file') - await fsUnlink(`${location}/${filteredName}`) + try { + await fsUnlink(`${location}/${filteredName}`) + } catch (err) { + throw _wrapError( + err, + 'failed to delete file', + { location, filteredName }, + WriteError + ) + } } async function deleteDirectory(location, name) { @@ -109,7 +128,16 @@ async function deleteDirectory(location, name) { logger.log({ location, filteredName }, 'deleting directory') - await rmrf(`${location}/${filteredName}`) + try { + await rmrf(`${location}/${filteredName}`) + } catch (err) { + throw _wrapError( + err, + 'failed to delete directory', + { location, filteredName }, + WriteError + ) + } } async function checkIfFileExists(location, name) { @@ -121,7 +149,12 @@ async function checkIfFileExists(location, name) { if (err.code === 'ENOENT') { return false } - throw new ReadError('failed to stat file').withCause(err) + throw _wrapError( + err, + 'failed to stat file', + { location, filteredName }, + ReadError + ) } } @@ -146,15 +179,31 @@ async function directorySize(location, name) { } } } catch (err) { - throw new ReadError({ - message: 'failed to get directory size', - info: { location, name } - }).withCause(err) + throw _wrapError( + err, + 'failed to get directory size', + { location, name }, + ReadError + ) } return size } +function _wrapError(error, message, params, ErrorType) { + if (error.code === 'ENOENT') { + return new NotFoundError({ + message: 'no such file or directory', + info: params + }).withCause(error) + } else { + return new ErrorType({ + message: message, + info: params + }).withCause(error) + } +} + module.exports = { sendFile: callbackify(sendFile), sendStream: callbackify(sendStream), diff --git a/services/filestore/test/unit/js/FSPersistorManagerTests.js b/services/filestore/test/unit/js/FSPersistorManagerTests.js index cb177989a5..d0bd6b078e 100644 --- a/services/filestore/test/unit/js/FSPersistorManagerTests.js +++ b/services/filestore/test/unit/js/FSPersistorManagerTests.js @@ -80,7 +80,7 @@ describe('FSPersistorManagerTests', function() { files[0], localFilesystemPath ) - ).to.eventually.be.rejectedWith(error) + ).to.eventually.be.rejected.and.have.property('cause', error) }) }) @@ -152,8 +152,8 @@ describe('FSPersistorManagerTests', function() { await expect( FSPersistorManager.promises.getFileStream(location, files[0], {}) ) - .to.eventually.be.rejectedWith('file not found') - .and.be.an.instanceOf(Errors.NotFoundError) + .to.eventually.be.rejected.and.be.an.instanceOf(Errors.NotFoundError) + .and.have.property('cause', err) }) it('should wrap any other error', async function() { @@ -234,7 +234,7 @@ describe('FSPersistorManagerTests', function() { fs.unlink.yields(error) await expect( FSPersistorManager.promises.deleteFile(location, files[0]) - ).to.eventually.be.rejectedWith(error) + ).to.eventually.be.rejected.and.have.property('cause', error) }) }) @@ -250,7 +250,7 @@ describe('FSPersistorManagerTests', function() { rimraf.yields(error) await expect( FSPersistorManager.promises.deleteDirectory(location, files[0]) - ).to.eventually.be.rejectedWith(error) + ).to.eventually.be.rejected.and.have.property('cause', error) }) }) From fccb28898554a98c90e3e5b1cc062761b38007df Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 6 Jan 2020 15:13:50 +0000 Subject: [PATCH 52/73] Add comment regarding deleteDirectory being internal-only --- services/filestore/app/js/FSPersistorManager.js | 1 + 1 file changed, 1 insertion(+) diff --git a/services/filestore/app/js/FSPersistorManager.js b/services/filestore/app/js/FSPersistorManager.js index c649dfb61b..1a9d2b824d 100644 --- a/services/filestore/app/js/FSPersistorManager.js +++ b/services/filestore/app/js/FSPersistorManager.js @@ -123,6 +123,7 @@ async function deleteFile(location, name) { } } +// this is only called internally for clean-up by `FileHandler` and isn't part of the external API async function deleteDirectory(location, name) { const filteredName = filterName(name.replace(/\/$/, '')) From 37d44d5d8655fd83179b60193bf5742e36b304a5 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 6 Jan 2020 15:58:57 +0000 Subject: [PATCH 53/73] Return full path in stubbed 'glob' method --- services/filestore/test/unit/js/FSPersistorManagerTests.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/filestore/test/unit/js/FSPersistorManagerTests.js b/services/filestore/test/unit/js/FSPersistorManagerTests.js index d0bd6b078e..6847f5b8ef 100644 --- a/services/filestore/test/unit/js/FSPersistorManagerTests.js +++ b/services/filestore/test/unit/js/FSPersistorManagerTests.js @@ -20,6 +20,7 @@ describe('FSPersistorManagerTests', function() { const error = new Error('guru meditation error') const files = ['animals/wombat.tex', 'vegetables/potato.tex'] + const globs = [`${location}/${files[0]}`, `${location}/${files[1]}`] const filteredFilenames = ['animals_wombat.tex', 'vegetables_potato.tex'] let fs, rimraf, stream, LocalFileWriter, FSPersistorManager, glob @@ -31,7 +32,7 @@ describe('FSPersistorManagerTests', function() { open: sinon.stub().yields(null, fd), stat: sinon.stub().yields(null, stat) } - glob = sinon.stub().yields(null, files) + glob = sinon.stub().yields(null, globs) rimraf = sinon.stub().yields() stream = { pipeline: sinon.stub().yields() } LocalFileWriter = { From ec80052d9a70935da8b75983f5a89a9e1426ec57 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Thu, 2 Jan 2020 15:51:09 +0000 Subject: [PATCH 54/73] Decaf cleanup of FileController Remove deprecated res.send --- services/filestore/app/js/FileController.js | 288 +++++++------- .../filestore/app/js/HealthCheckController.js | 4 +- .../test/unit/js/FileControllerTests.js | 361 +++++++++--------- 3 files changed, 320 insertions(+), 333 deletions(-) diff --git a/services/filestore/app/js/FileController.js b/services/filestore/app/js/FileController.js index ea41f3d813..dbba9a93cc 100644 --- a/services/filestore/app/js/FileController.js +++ b/services/filestore/app/js/FileController.js @@ -1,41 +1,37 @@ -/* eslint-disable - camelcase, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -let FileController const PersistorManager = require('./PersistorManager') -const settings = require('settings-sharelatex') const logger = require('logger-sharelatex') const FileHandler = require('./FileHandler') const metrics = require('metrics-sharelatex') const parseRange = require('range-parser') const Errors = require('./Errors') +const { pipeline } = require('stream') -const oneDayInSeconds = 60 * 60 * 24 const maxSizeInBytes = 1024 * 1024 * 1024 // 1GB -module.exports = FileController = { - getFile(req, res) { - const { key, bucket } = req - const { format, style } = req.query - const options = { - key, - bucket, - format, - style - } - metrics.inc('getFile') - logger.log({ key, bucket, format, style }, 'receiving request to get file') - if (req.headers.range != null) { - const range = FileController._get_range(req.headers.range) +module.exports = { + getFile, + getFileHead, + insertFile, + copyFile, + deleteFile, + directorySize +} + +function getFile(req, res) { + const { key, bucket } = req + const { format, style } = req.query + const options = { + key, + bucket, + format, + style + } + metrics.inc('getFile') + logger.log({ key, bucket, format, style }, 'receiving request to get file') + + if (req.headers.range) { + const range = _getRange(req.headers.range) + if (range) { options.start = range.start options.end = range.end logger.log( @@ -43,131 +39,131 @@ module.exports = FileController = { 'getting range of bytes from file' ) } - return FileHandler.getFile(bucket, key, options, function(err, fileStream) { - if (err != null) { - if (err instanceof Errors.NotFoundError) { - return res.send(404) - } else { - logger.err( - { err, key, bucket, format, style }, - 'problem getting file' - ) - return res.send(500) - } - } else if (req.query.cacheWarm) { - logger.log( - { key, bucket, format, style }, - 'request is only for cache warm so not sending stream' - ) - return res.send(200) - } else { - logger.log({ key, bucket, format, style }, 'sending file to response') - return fileStream.pipe(res) - } - }) - }, + } - getFileHead(req, res) { - const { key, bucket } = req - metrics.inc('getFileSize') - logger.log({ key, bucket }, 'receiving request to get file metadata') - return FileHandler.getFileSize(bucket, key, function(err, fileSize) { - if (err != null) { + FileHandler.getFile(bucket, key, options, function(err, fileStream) { + if (err) { + if (err instanceof Errors.NotFoundError) { + res.sendStatus(404) + } else { + logger.err({ err, key, bucket, format, style }, 'problem getting file') + res.sendStatus(500) + } + return + } + + if (req.query.cacheWarm) { + logger.log( + { key, bucket, format, style }, + 'request is only for cache warm so not sending stream' + ) + return res.sendStatus(200) + } + + logger.log({ key, bucket, format, style }, 'sending file to response') + pipeline(fileStream, res) + }) +} + +function getFileHead(req, res) { + const { key, bucket } = req + metrics.inc('getFileSize') + logger.log({ key, bucket }, 'receiving request to get file metadata') + FileHandler.getFileSize(bucket, key, function(err, fileSize) { + if (err) { + if (err instanceof Errors.NotFoundError) { + res.sendStatus(404) + } else { + res.sendStatus(500) + } + return + } + res.set('Content-Length', fileSize) + res.status(200).end() + }) +} + +function insertFile(req, res) { + metrics.inc('insertFile') + const { key, bucket } = req + logger.log({ key, bucket }, 'receiving request to insert file') + FileHandler.insertFile(bucket, key, req, function(err) { + if (err) { + logger.log({ err, key, bucket }, 'error inserting file') + res.sendStatus(500) + } else { + res.sendStatus(200) + } + }) +} + +function copyFile(req, res) { + metrics.inc('copyFile') + const { key, bucket } = req + const oldProjectId = req.body.source.project_id + const oldFileId = req.body.source.file_id + logger.log( + { key, bucket, oldProject_id: oldProjectId, oldFile_id: oldFileId }, + 'receiving request to copy file' + ) + + PersistorManager.copyFile( + bucket, + `${oldProjectId}/${oldFileId}`, + key, + function(err) { + if (err) { if (err instanceof Errors.NotFoundError) { - res.status(404).end() + res.sendStatus(404) } else { - res.status(500).end() + logger.log( + { err, oldProject_id: oldProjectId, oldFile_id: oldFileId }, + 'something went wrong copying file' + ) + res.sendStatus(500) } return } - res.set('Content-Length', fileSize) - return res.status(200).end() - }) - }, - insertFile(req, res) { - metrics.inc('insertFile') - const { key, bucket } = req - logger.log({ key, bucket }, 'receiving request to insert file') - return FileHandler.insertFile(bucket, key, req, function(err) { - if (err != null) { - logger.log({ err, key, bucket }, 'error inserting file') - return res.send(500) - } else { - return res.send(200) - } - }) - }, - - copyFile(req, res) { - metrics.inc('copyFile') - const { key, bucket } = req - const oldProject_id = req.body.source.project_id - const oldFile_id = req.body.source.file_id - logger.log( - { key, bucket, oldProject_id, oldFile_id }, - 'receiving request to copy file' - ) - return PersistorManager.copyFile( - bucket, - `${oldProject_id}/${oldFile_id}`, - key, - function(err) { - if (err != null) { - if (err instanceof Errors.NotFoundError) { - return res.send(404) - } else { - logger.log( - { err, oldProject_id, oldFile_id }, - 'something went wrong copying file' - ) - return res.send(500) - } - } else { - return res.send(200) - } - } - ) - }, - - deleteFile(req, res) { - metrics.inc('deleteFile') - const { key, bucket } = req - logger.log({ key, bucket }, 'receiving request to delete file') - return FileHandler.deleteFile(bucket, key, function(err) { - if (err != null) { - logger.log({ err, key, bucket }, 'something went wrong deleting file') - return res.send(500) - } else { - return res.send(204) - } - }) - }, - - _get_range(header) { - const parsed = parseRange(maxSizeInBytes, header) - if (parsed === -1 || parsed === -2 || parsed.type !== 'bytes') { - return null - } else { - const range = parsed[0] - return { start: range.start, end: range.end } + res.sendStatus(200) } - }, + ) +} - directorySize(req, res) { - metrics.inc('projectSize') - const { project_id, bucket } = req - logger.log({ project_id, bucket }, 'receiving request to project size') - return FileHandler.getDirectorySize(bucket, project_id, function( - err, - size - ) { - if (err != null) { - logger.log({ err, project_id, bucket }, 'error inserting file') - return res.send(500) - } else { - return res.json({ 'total bytes': size }) - } - }) +function deleteFile(req, res) { + metrics.inc('deleteFile') + const { key, bucket } = req + logger.log({ key, bucket }, 'receiving request to delete file') + return FileHandler.deleteFile(bucket, key, function(err) { + if (err != null) { + logger.log({ err, key, bucket }, 'something went wrong deleting file') + return res.sendStatus(500) + } else { + return res.sendStatus(204) + } + }) +} + +function directorySize(req, res) { + metrics.inc('projectSize') + const { project_id: projectId, bucket } = req + logger.log({ projectId, bucket }, 'receiving request to project size') + FileHandler.getDirectorySize(bucket, projectId, function(err, size) { + if (err) { + logger.log({ err, projectId, bucket }, 'error inserting file') + return res.sendStatus(500) + } + + res.json({ 'total bytes': size }) + }) +} + +function _getRange(header) { + const parsed = parseRange(maxSizeInBytes, header) + if (parsed === -1 || parsed === -2 || parsed.type !== 'bytes') { + return null + } else { + const range = parsed[0] + return { start: range.start, end: range.end } } } diff --git a/services/filestore/app/js/HealthCheckController.js b/services/filestore/app/js/HealthCheckController.js index 8d6e35b783..5e12469ad3 100644 --- a/services/filestore/app/js/HealthCheckController.js +++ b/services/filestore/app/js/HealthCheckController.js @@ -63,10 +63,10 @@ module.exports = { check(req, res) { logger.log({}, 'performing health check') Promise.all([checkCanGetFiles(), checkFileConvert()]) - .then(() => res.send(200)) + .then(() => res.sendStatus(200)) .catch(err => { logger.err({ err }, 'Health check: error running') - res.send(500) + res.sendStatus(500) }) } } diff --git a/services/filestore/test/unit/js/FileControllerTests.js b/services/filestore/test/unit/js/FileControllerTests.js index 2505e87c9b..00e3fd1505 100644 --- a/services/filestore/test/unit/js/FileControllerTests.js +++ b/services/filestore/test/unit/js/FileControllerTests.js @@ -1,52 +1,60 @@ -/* eslint-disable - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const { assert } = require('chai') const sinon = require('sinon') const chai = require('chai') -const should = chai.should() const { expect } = chai -const modulePath = '../../../app/js/FileController.js' const SandboxedModule = require('sandboxed-module') +const Errors = require('../../../app/js/Errors') +const modulePath = '../../../app/js/FileController.js' describe('FileController', function() { - beforeEach(function() { - this.PersistorManager = { - sendStream: sinon.stub(), - copyFile: sinon.stub(), - deleteFile: sinon.stub() - } - - this.settings = { - s3: { - buckets: { - user_files: 'user_files' - } + let PersistorManager, + FileHandler, + LocalFileWriter, + FileController, + req, + res, + stream + const settings = { + s3: { + buckets: { + user_files: 'user_files' } } - this.FileHandler = { - getFile: sinon.stub(), - getFileSize: sinon.stub(), - deleteFile: sinon.stub(), - insertFile: sinon.stub(), - getDirectorySize: sinon.stub() + } + const fileSize = 1234 + const fileStream = 'fileStream' + const projectId = 'projectId' + const fileId = 'file_id' + const bucket = 'user_files' + const key = `${projectId}/${fileId}` + + beforeEach(function() { + PersistorManager = { + sendStream: sinon.stub().yields(), + copyFile: sinon.stub().yields(), + deleteFile: sinon.stub().yields() } - this.LocalFileWriter = {} - this.controller = SandboxedModule.require(modulePath, { + + FileHandler = { + getFile: sinon.stub().yields(null, fileStream), + getFileSize: sinon.stub().yields(null, fileSize), + deleteFile: sinon.stub().yields(), + insertFile: sinon.stub().yields(), + getDirectorySize: sinon.stub().yields(null, fileSize) + } + + LocalFileWriter = {} + stream = { + pipeline: sinon.stub() + } + + FileController = SandboxedModule.require(modulePath, { requires: { - './LocalFileWriter': this.LocalFileWriter, - './FileHandler': this.FileHandler, - './PersistorManager': this.PersistorManager, - './Errors': (this.Errors = { NotFoundError: sinon.stub() }), - 'settings-sharelatex': this.settings, + './LocalFileWriter': LocalFileWriter, + './FileHandler': FileHandler, + './PersistorManager': PersistorManager, + './Errors': Errors, + stream: stream, + 'settings-sharelatex': settings, 'metrics-sharelatex': { inc() {} }, @@ -54,244 +62,227 @@ describe('FileController', function() { log() {}, err() {} } - } + }, + globals: { console } }) - this.project_id = 'project_id' - this.file_id = 'file_id' - this.bucket = 'user_files' - this.key = `${this.project_id}/${this.file_id}` - this.req = { - key: this.key, - bucket: this.bucket, + + req = { + key: key, + bucket: bucket, query: {}, params: { - project_id: this.project_id, - file_id: this.file_id + project_id: projectId, + file_id: fileId }, headers: {} } - this.res = { + + res = { set: sinon.stub().returnsThis(), + sendStatus: sinon.stub().returnsThis(), status: sinon.stub().returnsThis() } - return (this.fileStream = {}) }) describe('getFile', function() { - it('should pipe the stream', function(done) { - this.FileHandler.getFile.callsArgWith(3, null, this.fileStream) - this.fileStream.pipe = res => { - res.should.equal(this.res) - return done() - } - return this.controller.getFile(this.req, this.res) + it('should pipe the stream', function() { + FileController.getFile(req, res) + expect(stream.pipeline).to.have.been.calledWith(fileStream, res) }) it('should send a 200 if the cacheWarm param is true', function(done) { - this.req.query.cacheWarm = true - this.FileHandler.getFile.callsArgWith(3, null, this.fileStream) - this.res.send = statusCode => { + req.query.cacheWarm = true + res.sendStatus = statusCode => { statusCode.should.equal(200) - return done() + done() } - return this.controller.getFile(this.req, this.res) + FileController.getFile(req, res) }) it('should send a 500 if there is a problem', function(done) { - this.FileHandler.getFile.callsArgWith(3, 'error') - this.res.send = code => { + FileHandler.getFile.yields('error') + res.sendStatus = code => { code.should.equal(500) - return done() + done() } - return this.controller.getFile(this.req, this.res) + FileController.getFile(req, res) }) - return describe("with a 'Range' header set", function() { + describe('with a range header', function() { + let expectedOptions + beforeEach(function() { - return (this.req.headers.range = 'bytes=0-8') + expectedOptions = { + bucket, + key, + format: undefined, + style: undefined + } }) - return it("should pass 'start' and 'end' options to FileHandler", function(done) { - this.FileHandler.getFile.callsArgWith(3, null, this.fileStream) - this.fileStream.pipe = res => { - expect(this.FileHandler.getFile.lastCall.args[2].start).to.equal(0) - expect(this.FileHandler.getFile.lastCall.args[2].end).to.equal(8) - return done() - } - return this.controller.getFile(this.req, this.res) + it('should pass range options to FileHandler', function() { + req.headers.range = 'bytes=0-8' + expectedOptions.start = 0 + expectedOptions.end = 8 + + FileController.getFile(req, res) + expect(FileHandler.getFile).to.have.been.calledWith( + bucket, + key, + expectedOptions + ) + }) + + it('should ignore an invalid range header', function() { + req.headers.range = 'potato' + FileController.getFile(req, res) + expect(FileHandler.getFile).to.have.been.calledWith( + bucket, + key, + expectedOptions + ) + }) + + it("should ignore any type other than 'bytes'", function() { + req.headers.range = 'wombats=0-8' + FileController.getFile(req, res) + expect(FileHandler.getFile).to.have.been.calledWith( + bucket, + key, + expectedOptions + ) }) }) }) describe('getFileHead', function() { it('should return the file size in a Content-Length header', function(done) { - const expectedFileSize = 84921 - this.FileHandler.getFileSize.yields( - new Error('FileHandler.getFileSize: unexpected arguments') - ) - this.FileHandler.getFileSize - .withArgs(this.bucket, this.key) - .yields(null, expectedFileSize) - - this.res.end = () => { - expect(this.res.status.lastCall.args[0]).to.equal(200) - expect( - this.res.set.calledWith('Content-Length', expectedFileSize) - ).to.equal(true) - return done() + res.end = () => { + expect(res.status).to.have.been.calledWith(200) + expect(res.set).to.have.been.calledWith('Content-Length', fileSize) + done() } - return this.controller.getFileHead(this.req, this.res) + FileController.getFileHead(req, res) }) it('should return a 404 is the file is not found', function(done) { - this.FileHandler.getFileSize.yields(new this.Errors.NotFoundError()) + FileHandler.getFileSize.yields(new Errors.NotFoundError()) - this.res.end = () => { - expect(this.res.status.lastCall.args[0]).to.equal(404) - return done() + res.sendStatus = code => { + expect(code).to.equal(404) + done() } - return this.controller.getFileHead(this.req, this.res) + FileController.getFileHead(req, res) }) - return it('should return a 500 on internal errors', function(done) { - this.FileHandler.getFileSize.yields(new Error()) + it('should return a 500 on internal errors', function(done) { + FileHandler.getFileSize.yields(new Error()) - this.res.end = () => { - expect(this.res.status.lastCall.args[0]).to.equal(500) - return done() + res.sendStatus = code => { + expect(code).to.equal(500) + done() } - return this.controller.getFileHead(this.req, this.res) + FileController.getFileHead(req, res) }) }) - describe('insertFile', () => + describe('insertFile', function() { it('should send bucket name key and res to PersistorManager', function(done) { - this.FileHandler.insertFile.callsArgWith(3) - this.res.send = () => { - this.FileHandler.insertFile - .calledWith(this.bucket, this.key, this.req) - .should.equal(true) - return done() + res.sendStatus = code => { + expect(FileHandler.insertFile).to.have.been.calledWith(bucket, key, req) + expect(code).to.equal(200) + done() } - return this.controller.insertFile(this.req, this.res) - })) + FileController.insertFile(req, res) + }) + }) describe('copyFile', function() { + const oldFileId = 'oldFileId' + const oldProjectId = 'oldProjectid' + const oldKey = `${oldProjectId}/${oldFileId}` + beforeEach(function() { - this.oldFile_id = 'old_file_id' - this.oldProject_id = 'old_project_id' - return (this.req.body = { + req.body = { source: { - project_id: this.oldProject_id, - file_id: this.oldFile_id + project_id: oldProjectId, + file_id: oldFileId } - }) + } }) it('should send bucket name and both keys to PersistorManager', function(done) { - this.PersistorManager.copyFile.callsArgWith(3) - this.res.send = code => { + res.sendStatus = code => { code.should.equal(200) - this.PersistorManager.copyFile - .calledWith( - this.bucket, - `${this.oldProject_id}/${this.oldFile_id}`, - this.key - ) - .should.equal(true) - return done() + expect(PersistorManager.copyFile).to.have.been.calledWith( + bucket, + oldKey, + key + ) + done() } - return this.controller.copyFile(this.req, this.res) + FileController.copyFile(req, res) }) it('should send a 404 if the original file was not found', function(done) { - this.PersistorManager.copyFile.callsArgWith( - 3, - new this.Errors.NotFoundError() - ) - this.res.send = code => { + PersistorManager.copyFile.yields(new Errors.NotFoundError()) + res.sendStatus = code => { code.should.equal(404) - return done() + done() } - return this.controller.copyFile(this.req, this.res) + FileController.copyFile(req, res) }) - return it('should send a 500 if there was an error', function(done) { - this.PersistorManager.copyFile.callsArgWith(3, 'error') - this.res.send = code => { + it('should send a 500 if there was an error', function(done) { + PersistorManager.copyFile.yields('error') + res.sendStatus = code => { code.should.equal(500) - return done() + done() } - return this.controller.copyFile(this.req, this.res) + FileController.copyFile(req, res) }) }) describe('delete file', function() { it('should tell the file handler', function(done) { - this.FileHandler.deleteFile.callsArgWith(2) - this.res.send = code => { + res.sendStatus = code => { code.should.equal(204) - this.FileHandler.deleteFile - .calledWith(this.bucket, this.key) - .should.equal(true) - return done() + expect(FileHandler.deleteFile).to.have.been.calledWith(bucket, key) + done() } - return this.controller.deleteFile(this.req, this.res) + FileController.deleteFile(req, res) }) - return it('should send a 500 if there was an error', function(done) { - this.FileHandler.deleteFile.callsArgWith(2, 'error') - this.res.send = function(code) { + it('should send a 500 if there was an error', function(done) { + FileHandler.deleteFile.yields('error') + res.sendStatus = code => { code.should.equal(500) - return done() + done() } - return this.controller.deleteFile(this.req, this.res) + FileController.deleteFile(req, res) }) }) - describe('_get_range', function() { - it('should parse a valid Range header', function(done) { - const result = this.controller._get_range('bytes=0-200') - expect(result).to.not.equal(null) - expect(result.start).to.equal(0) - expect(result.end).to.equal(200) - return done() - }) - - it('should return null for an invalid Range header', function(done) { - const result = this.controller._get_range('wat') - expect(result).to.equal(null) - return done() - }) - - return it("should return null for any type other than 'bytes'", function(done) { - const result = this.controller._get_range('carrots=0-200') - expect(result).to.equal(null) - return done() - }) - }) - - return describe('directorySize', function() { + describe('directorySize', function() { it('should return total directory size bytes', function(done) { - this.FileHandler.getDirectorySize.callsArgWith(2, null, 1024) - return this.controller.directorySize(this.req, { + FileController.directorySize(req, { json: result => { - expect(result['total bytes']).to.equal(1024) - return done() + expect(result['total bytes']).to.equal(fileSize) + done() } }) }) - return it('should send a 500 if there was an error', function(done) { - this.FileHandler.getDirectorySize.callsArgWith(2, 'error') - this.res.send = function(code) { + it('should send a 500 if there was an error', function(done) { + FileHandler.getDirectorySize.callsArgWith(2, 'error') + res.sendStatus = code => { code.should.equal(500) - return done() + done() } - return this.controller.directorySize(this.req, this.res) + FileController.directorySize(req, res) }) }) }) From 10f708791237c1e3fc166a404751a0df86b01368 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Thu, 2 Jan 2020 16:12:07 +0000 Subject: [PATCH 55/73] Post-decaf cleanup of KeyBuilderTests --- .../filestore/test/unit/js/KeybuilderTests.js | 48 +++++++------------ 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/services/filestore/test/unit/js/KeybuilderTests.js b/services/filestore/test/unit/js/KeybuilderTests.js index 09a0ea8717..5271e892ed 100644 --- a/services/filestore/test/unit/js/KeybuilderTests.js +++ b/services/filestore/test/unit/js/KeybuilderTests.js @@ -1,26 +1,13 @@ -/* eslint-disable - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ - -const { assert } = require('chai') -const sinon = require('sinon') -const chai = require('chai') -const should = chai.should() -const { expect } = chai -const modulePath = '../../../app/js/KeyBuilder.js' const SandboxedModule = require('sandboxed-module') +const modulePath = '../../../app/js/KeyBuilder.js' + describe('LocalFileWriter', function() { + let KeyBuilder + const key = 'wombat/potato' + beforeEach(function() { - this.keyBuilder = SandboxedModule.require(modulePath, { + KeyBuilder = SandboxedModule.require(modulePath, { requires: { 'logger-sharelatex': { log() {}, @@ -28,31 +15,28 @@ describe('LocalFileWriter', function() { } } }) - return (this.key = '123/456') }) - return describe('cachedKey', function() { - it('should add the fomat on', function() { + describe('cachedKey', function() { + it('should add the format to the key', function() { const opts = { format: 'png' } - const newKey = this.keyBuilder.addCachingToKey(this.key, opts) - return newKey.should.equal(`${this.key}-converted-cache/format-png`) + const newKey = KeyBuilder.addCachingToKey(key, opts) + newKey.should.equal(`${key}-converted-cache/format-png`) }) - it('should add the style on', function() { + it('should add the style to the key', function() { const opts = { style: 'thumbnail' } - const newKey = this.keyBuilder.addCachingToKey(this.key, opts) - return newKey.should.equal(`${this.key}-converted-cache/style-thumbnail`) + const newKey = KeyBuilder.addCachingToKey(key, opts) + newKey.should.equal(`${key}-converted-cache/style-thumbnail`) }) - return it('should add format on first', function() { + it('should add format first, then style', function() { const opts = { style: 'thumbnail', format: 'png' } - const newKey = this.keyBuilder.addCachingToKey(this.key, opts) - return newKey.should.equal( - `${this.key}-converted-cache/format-png-style-thumbnail` - ) + const newKey = KeyBuilder.addCachingToKey(key, opts) + newKey.should.equal(`${key}-converted-cache/format-png-style-thumbnail`) }) }) }) From dbfacce98822036574945451250b1f71b2537403 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Thu, 2 Jan 2020 16:37:47 +0000 Subject: [PATCH 56/73] Post-decaf cleanup of PersistorManager --- services/filestore/app/js/PersistorManager.js | 50 ++--- .../test/unit/js/PersistorManagerTests.js | 185 ++++++------------ 2 files changed, 74 insertions(+), 161 deletions(-) diff --git a/services/filestore/app/js/PersistorManager.js b/services/filestore/app/js/PersistorManager.js index 182e39b085..f8ca7b9d2c 100644 --- a/services/filestore/app/js/PersistorManager.js +++ b/services/filestore/app/js/PersistorManager.js @@ -1,37 +1,19 @@ -// TODO: This file was created by bulk-decaffeinate. -// Sanity-check the conversion and remove this comment. -/* - * decaffeinate suggestions: - * DS103: Rewrite code to no longer use __guard__ - * DS205: Consider reworking code to avoid use of IIFEs - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ const settings = require('settings-sharelatex') const logger = require('logger-sharelatex') -// assume s3 if none specified -__guard__( - settings != null ? settings.filestore : undefined, - x => x.backend || (settings.filestore.backend = 's3') -) +module.exports = (function() { + logger.log( + { + backend: settings.filestore.backend + }, + 'Loading backend' + ) -logger.log( - { - backend: __guard__( - settings != null ? settings.filestore : undefined, - x1 => x1.backend - ) - }, - 'Loading backend' -) -module.exports = (() => { - switch ( - __guard__( - settings != null ? settings.filestore : undefined, - x2 => x2.backend - ) - ) { + if (!settings.filestore.backend) { + throw new Error('no backend specified - config incomplete') + } + + switch (settings.filestore.backend) { case 'aws-sdk': return require('./AWSSDKPersistorManager') case 's3': @@ -40,13 +22,7 @@ module.exports = (() => { return require('./FSPersistorManager') default: throw new Error( - `Unknown filestore backend: ${settings.filestore.backend}` + `unknown filestore backend: ${settings.filestore.backend}` ) } })() - -function __guard__(value, transform) { - return typeof value !== 'undefined' && value !== null - ? transform(value) - : undefined -} diff --git a/services/filestore/test/unit/js/PersistorManagerTests.js b/services/filestore/test/unit/js/PersistorManagerTests.js index ff49c05ce9..d8fd887265 100644 --- a/services/filestore/test/unit/js/PersistorManagerTests.js +++ b/services/filestore/test/unit/js/PersistorManagerTests.js @@ -1,137 +1,74 @@ -/* eslint-disable - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const logger = require('logger-sharelatex') -const { assert } = require('chai') const sinon = require('sinon') const chai = require('chai') -const should = chai.should() const { expect } = chai -const modulePath = '../../../app/js/PersistorManager.js' const SandboxedModule = require('sandboxed-module') -describe('PersistorManagerTests', function() { +const modulePath = '../../../app/js/PersistorManager.js' + +describe('PersistorManager', function() { + let PersistorManager, + FSPersistorManager, + S3PersistorManager, + settings, + requires + beforeEach(function() { - return (this.S3PersistorManager = { - getFileStream: sinon.stub(), - checkIfFileExists: sinon.stub(), - deleteFile: sinon.stub(), - deleteDirectory: sinon.stub(), - sendStream: sinon.stub(), - insertFile: sinon.stub() - }) + FSPersistorManager = { + wrappedMethod: sinon.stub().returns('FSPersistorManager') + } + S3PersistorManager = { + wrappedMethod: sinon.stub().returns('S3PersistorManager') + } + + settings = { + filestore: {} + } + + requires = { + './S3PersistorManager': S3PersistorManager, + './FSPersistorManager': FSPersistorManager, + 'settings-sharelatex': settings, + 'logger-sharelatex': { + log() {}, + err() {} + } + } }) - describe('test s3 mixin', function() { - beforeEach(function() { - this.settings = { - filestore: { - backend: 's3' - } - } - this.requires = { - './S3PersistorManager': this.S3PersistorManager, - 'settings-sharelatex': this.settings, - 'logger-sharelatex': { - log() {}, - err() {} - } - } - return (this.PersistorManager = SandboxedModule.require(modulePath, { - requires: this.requires - })) - }) + it('should implement the S3 wrapped method when S3 is configured', function() { + settings.filestore.backend = 's3' + PersistorManager = SandboxedModule.require(modulePath, { requires }) - it('should load getFileStream', function(done) { - this.PersistorManager.should.respondTo('getFileStream') - this.PersistorManager.getFileStream() - this.S3PersistorManager.getFileStream.calledOnce.should.equal(true) - return done() - }) - - it('should load checkIfFileExists', function(done) { - this.PersistorManager.checkIfFileExists() - this.S3PersistorManager.checkIfFileExists.calledOnce.should.equal(true) - return done() - }) - - it('should load deleteFile', function(done) { - this.PersistorManager.deleteFile() - this.S3PersistorManager.deleteFile.calledOnce.should.equal(true) - return done() - }) - - it('should load deleteDirectory', function(done) { - this.PersistorManager.deleteDirectory() - this.S3PersistorManager.deleteDirectory.calledOnce.should.equal(true) - return done() - }) - - it('should load sendStream', function(done) { - this.PersistorManager.sendStream() - this.S3PersistorManager.sendStream.calledOnce.should.equal(true) - return done() - }) - - return it('should load insertFile', function(done) { - this.PersistorManager.insertFile() - this.S3PersistorManager.insertFile.calledOnce.should.equal(true) - return done() - }) + expect(PersistorManager).to.respondTo('wrappedMethod') + expect(PersistorManager.wrappedMethod()).to.equal('S3PersistorManager') }) - describe('test unspecified mixins', () => - it('should load s3 when no wrapper specified', function(done) { - this.settings = { filestore: {} } - this.requires = { - './S3PersistorManager': this.S3PersistorManager, - 'settings-sharelatex': this.settings, - 'logger-sharelatex': { - log() {}, - err() {} - } - } - this.PersistorManager = SandboxedModule.require(modulePath, { - requires: this.requires - }) - this.PersistorManager.should.respondTo('getFileStream') - this.PersistorManager.getFileStream() - this.S3PersistorManager.getFileStream.calledOnce.should.equal(true) - return done() - })) + it('should implement the FS wrapped method when FS is configured', function() { + settings.filestore.backend = 'fs' + PersistorManager = SandboxedModule.require(modulePath, { requires }) - return describe('test invalid mixins', () => - it('should not load an invalid wrapper', function(done) { - this.settings = { - filestore: { - backend: 'magic' - } - } - this.requires = { - './S3PersistorManager': this.S3PersistorManager, - 'settings-sharelatex': this.settings, - 'logger-sharelatex': { - log() {}, - err() {} - } - } - this.fsWrapper = null - try { - this.PersistorManager = SandboxedModule.require(modulePath, { - requires: this.requires - }) - } catch (error) { - assert.equal('Unknown filestore backend: magic', error.message) - } - assert.isNull(this.fsWrapper) - return done() - })) + expect(PersistorManager).to.respondTo('wrappedMethod') + expect(PersistorManager.wrappedMethod()).to.equal('FSPersistorManager') + }) + + it('should throw an error when the backend is not configured', function() { + try { + SandboxedModule.require(modulePath, { requires }) + } catch (err) { + expect(err.message).to.equal('no backend specified - config incomplete') + return + } + expect('should have caught an error').not.to.exist + }) + + it('should throw an error when the backend is unknown', function() { + settings.filestore.backend = 'magic' + try { + SandboxedModule.require(modulePath, { requires }) + } catch (err) { + expect(err.message).to.equal('unknown filestore backend: magic') + return + } + expect('should have caught an error').not.to.exist + }) }) From 0329c759dc99f836e511e9c38c4a1e1410385e22 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Sat, 4 Jan 2020 17:27:12 +0000 Subject: [PATCH 57/73] Remove wrapper function in PersistorManager --- services/filestore/app/js/PersistorManager.js | 46 +++++++++---------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/services/filestore/app/js/PersistorManager.js b/services/filestore/app/js/PersistorManager.js index f8ca7b9d2c..8124d66101 100644 --- a/services/filestore/app/js/PersistorManager.js +++ b/services/filestore/app/js/PersistorManager.js @@ -1,28 +1,26 @@ const settings = require('settings-sharelatex') const logger = require('logger-sharelatex') -module.exports = (function() { - logger.log( - { - backend: settings.filestore.backend - }, - 'Loading backend' - ) +logger.log( + { + backend: settings.filestore.backend + }, + 'Loading backend' +) +if (!settings.filestore.backend) { + throw new Error('no backend specified - config incomplete') +} - if (!settings.filestore.backend) { - throw new Error('no backend specified - config incomplete') - } - - switch (settings.filestore.backend) { - case 'aws-sdk': - return require('./AWSSDKPersistorManager') - case 's3': - return require('./S3PersistorManager') - case 'fs': - return require('./FSPersistorManager') - default: - throw new Error( - `unknown filestore backend: ${settings.filestore.backend}` - ) - } -})() +switch (settings.filestore.backend) { + case 'aws-sdk': + module.exports = require('./AWSSDKPersistorManager') + break + case 's3': + module.exports = require('./S3PersistorManager') + break + case 'fs': + module.exports = require('./FSPersistorManager') + break + default: + throw new Error(`unknown filestore backend: ${settings.filestore.backend}`) +} From fce275e1d4b7864b3d64ee238aeb7b4d26d3af3e Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Fri, 3 Jan 2020 10:06:19 +0000 Subject: [PATCH 58/73] Post-decaf cleanup of app.js --- services/filestore/app.js | 125 ++++-------------- services/filestore/app/js/ExceptionHandler.js | 97 ++++++++++++++ .../test/acceptance/js/FilestoreApp.js | 1 + 3 files changed, 127 insertions(+), 96 deletions(-) create mode 100644 services/filestore/app/js/ExceptionHandler.js diff --git a/services/filestore/app.js b/services/filestore/app.js index 9e76107ea6..d80514738c 100644 --- a/services/filestore/app.js +++ b/services/filestore/app.js @@ -1,89 +1,40 @@ -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -/* eslint-disable node/no-deprecated-api */ const Metrics = require('metrics-sharelatex') +const logger = require('logger-sharelatex') + Metrics.initialize('filestore') +logger.initialize('filestore') + +const settings = require('settings-sharelatex') const express = require('express') const bodyParser = require('body-parser') -let logger = require('logger-sharelatex') -logger.initialize('filestore') -const settings = require('settings-sharelatex') + const fileController = require('./app/js/FileController') const bucketController = require('./app/js/BucketController') const keyBuilder = require('./app/js/KeyBuilder') const healthCheckController = require('./app/js/HealthCheckController') -const domain = require('domain') -let appIsOk = true -const app = express() +const ExceptionHandler = require('./app/js/ExceptionHandler') +const exceptionHandler = new ExceptionHandler() -if ((settings.sentry != null ? settings.sentry.dsn : undefined) != null) { +const app = express() +app.exceptionHandler = exceptionHandler + +if (settings.sentry && settings.sentry.dsn) { logger.initializeErrorReporting(settings.sentry.dsn) } Metrics.open_sockets.monitor(logger) -if (Metrics.event_loop != null) { +Metrics.memory.monitor(logger) +if (Metrics.event_loop) { Metrics.event_loop.monitor(logger) } -Metrics.memory.monitor(logger) app.use(Metrics.http.monitor(logger)) - app.use(function(req, res, next) { Metrics.inc('http-request') - return next() + next() }) -app.use(function(req, res, next) { - const requestDomain = domain.create() - requestDomain.add(req) - requestDomain.add(res) - requestDomain.on('error', function(err) { - try { - // request a shutdown to prevent memory leaks - beginShutdown() - if (!res.headerSent) { - res.send(500, 'uncaught exception') - } - logger = require('logger-sharelatex') - req = { - body: req.body, - headers: req.headers, - url: req.url, - key: req.key, - statusCode: req.statusCode - } - err = { - message: err.message, - stack: err.stack, - name: err.name, - type: err.type, - arguments: err.arguments - } - return logger.err( - { err, req, res }, - 'uncaught exception thrown on request' - ) - } catch (exception) { - return logger.err( - { err: exception }, - 'exception in request domain handler' - ) - } - }) - return requestDomain.run(next) -}) - -app.use(function(req, res, next) { - if (!appIsOk) { - // when shutting down, close any HTTP keep-alive connections - res.set('Connection', 'close') - } - return next() -}) +exceptionHandler.addMiddleware(app) Metrics.injectMetricsRoute(app) @@ -108,7 +59,7 @@ app.put( bodyParser.json(), fileController.copyFile ) -app.del( +app.delete( '/project/:project_id/file/:file_id', keyBuilder.userFileKeyMiddleware, fileController.deleteFile @@ -156,7 +107,7 @@ app.put( bodyParser.json(), fileController.copyFile ) -app.del( +app.delete( '/project/:project_id/public/:public_file_id', keyBuilder.publicFileKeyMiddleware, fileController.deleteFile @@ -183,68 +134,50 @@ app.get('/heapdump', (req, res, next) => ) app.post('/shutdown', function(req, res) { - appIsOk = false - return res.send() + exceptionHandler.setNotOk() + res.sendStatus(200) }) app.get('/status', function(req, res) { - if (appIsOk) { - return res.send('filestore sharelatex up') + if (exceptionHandler.appIsOk()) { + res.send('filestore sharelatex up') } else { logger.log('app is not ok - shutting down') - return res.send('server is being shut down', 500) + res.send('server is being shut down').status(500) } }) app.get('/health_check', healthCheckController.check) -app.get('*', (req, res) => res.send(404)) - -var beginShutdown = function() { - if (appIsOk) { - appIsOk = false - // hard-terminate this process if graceful shutdown fails - const killTimer = setTimeout(() => process.exit(1), 120 * 1000) - if (typeof killTimer.unref === 'function') { - killTimer.unref() - } // prevent timer from keeping process alive - server.close(function() { - logger.log('closed all connections') - Metrics.close() - return typeof process.disconnect === 'function' - ? process.disconnect() - : undefined - }) - return logger.log('server will stop accepting connections') - } -} +app.get('*', (req, res) => res.sendStatus(404)) const port = settings.internal.filestore.port || 3009 const host = '0.0.0.0' if (!module.parent) { // Called directly - var server = app.listen(port, host, error => { + const server = app.listen(port, host, error => { if (error) { logger.error('Error starting Filestore', error) throw error } logger.info(`Filestore starting up, listening on ${host}:${port}`) }) + exceptionHandler.server = server } module.exports = app process.on('SIGTERM', function() { logger.log('filestore got SIGTERM, shutting down gracefully') - return beginShutdown() + exceptionHandler.beginShutdown() }) -if (global.gc != null) { +if (global.gc) { const oneMinute = 60 * 1000 const gcTimer = setInterval(function() { global.gc() - return logger.log(process.memoryUsage(), 'global.gc') + logger.log(process.memoryUsage(), 'global.gc') }, 3 * oneMinute) gcTimer.unref() } diff --git a/services/filestore/app/js/ExceptionHandler.js b/services/filestore/app/js/ExceptionHandler.js new file mode 100644 index 0000000000..122e78805c --- /dev/null +++ b/services/filestore/app/js/ExceptionHandler.js @@ -0,0 +1,97 @@ +const Metrics = require('metrics-sharelatex') +const logger = require('logger-sharelatex') + +// TODO: domain has been deprecated for some time - do we need it and is there a better way? + +// eslint-disable-next-line node/no-deprecated-api +const domain = require('domain') + +const TWO_MINUTES = 120 * 1000 + +class ExceptionHandler { + constructor() { + this._appIsOk = true + } + + beginShutdown() { + if (this._appIsOk) { + this._appIsOk = false + + // hard-terminate this process if graceful shutdown fails + const killTimer = setTimeout(() => process.exit(1), TWO_MINUTES) + + if (typeof killTimer.unref === 'function') { + killTimer.unref() + } // prevent timer from keeping process alive + + this.server.close(function() { + logger.log('closed all connections') + Metrics.close() + if (typeof process.disconnect === 'function') { + process.disconnect() + } + }) + logger.log('server will stop accepting connections') + } + } + + addMiddleware(app) { + app.use(this.middleware.bind(this)) + } + + appIsOk() { + return this._appIsOk + } + + setNotOk() { + this._appIsOk = false + } + + middleware(req, res, next) { + const rescueLogger = require('logger-sharelatex') + const requestDomain = domain.create() + requestDomain.add(req) + requestDomain.add(res) + requestDomain.on('error', err => { + try { + // request a shutdown to prevent memory leaks + this.beginShutdown() + if (!res.headerSent) { + res.send('uncaught exception').status(500) + } + req = { + body: req.body, + headers: req.headers, + url: req.url, + key: req.key, + statusCode: req.statusCode + } + err = { + message: err.message, + stack: err.stack, + name: err.name, + type: err.type, + arguments: err.arguments + } + rescueLogger.err( + { err, req, res }, + 'uncaught exception thrown on request' + ) + } catch (exception) { + rescueLogger.err( + { err: exception }, + 'exception in request domain handler' + ) + } + }) + + if (!this._appIsOk) { + // when shutting down, close any HTTP keep-alive connections + res.set('Connection', 'close') + } + + requestDomain.run(next) + } +} + +module.exports = ExceptionHandler diff --git a/services/filestore/test/acceptance/js/FilestoreApp.js b/services/filestore/test/acceptance/js/FilestoreApp.js index 718d53bcf8..32ec7a5adf 100644 --- a/services/filestore/test/acceptance/js/FilestoreApp.js +++ b/services/filestore/test/acceptance/js/FilestoreApp.js @@ -44,6 +44,7 @@ class FilestoreApp { resolve() } ) + this.app.exceptionHandler.server = this.server }) if (Settings.filestore.backend === 's3') { From 6a679023d30bc13cbc9fb18a1ba96f7547363cd3 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Sat, 4 Jan 2020 17:15:21 +0000 Subject: [PATCH 59/73] Fix order of .status().send() --- services/filestore/app.js | 2 +- services/filestore/app/js/ExceptionHandler.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/filestore/app.js b/services/filestore/app.js index d80514738c..1e84440eb6 100644 --- a/services/filestore/app.js +++ b/services/filestore/app.js @@ -143,7 +143,7 @@ app.get('/status', function(req, res) { res.send('filestore sharelatex up') } else { logger.log('app is not ok - shutting down') - res.send('server is being shut down').status(500) + res.status(500).send('server is being shut down') } }) diff --git a/services/filestore/app/js/ExceptionHandler.js b/services/filestore/app/js/ExceptionHandler.js index 122e78805c..d52c00bb2c 100644 --- a/services/filestore/app/js/ExceptionHandler.js +++ b/services/filestore/app/js/ExceptionHandler.js @@ -57,7 +57,7 @@ class ExceptionHandler { // request a shutdown to prevent memory leaks this.beginShutdown() if (!res.headerSent) { - res.send('uncaught exception').status(500) + res.status(500).send('uncaught exception') } req = { body: req.body, From 85d3c0a852d1ebc6b0118dc7cc477b87bb0fc13d Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 6 Jan 2020 15:43:24 +0000 Subject: [PATCH 60/73] Remove old exception-handling and shutdown-related mechanisms --- services/filestore/app.js | 36 +------ services/filestore/app/js/ExceptionHandler.js | 97 ------------------- .../test/acceptance/js/FilestoreApp.js | 1 - 3 files changed, 2 insertions(+), 132 deletions(-) delete mode 100644 services/filestore/app/js/ExceptionHandler.js diff --git a/services/filestore/app.js b/services/filestore/app.js index 1e84440eb6..232c5b24bc 100644 --- a/services/filestore/app.js +++ b/services/filestore/app.js @@ -12,11 +12,8 @@ const fileController = require('./app/js/FileController') const bucketController = require('./app/js/BucketController') const keyBuilder = require('./app/js/KeyBuilder') const healthCheckController = require('./app/js/HealthCheckController') -const ExceptionHandler = require('./app/js/ExceptionHandler') -const exceptionHandler = new ExceptionHandler() const app = express() -app.exceptionHandler = exceptionHandler if (settings.sentry && settings.sentry.dsn) { logger.initializeErrorReporting(settings.sentry.dsn) @@ -34,8 +31,6 @@ app.use(function(req, res, next) { next() }) -exceptionHandler.addMiddleware(app) - Metrics.injectMetricsRoute(app) app.head( @@ -133,51 +128,24 @@ app.get('/heapdump', (req, res, next) => ) ) -app.post('/shutdown', function(req, res) { - exceptionHandler.setNotOk() - res.sendStatus(200) -}) - app.get('/status', function(req, res) { - if (exceptionHandler.appIsOk()) { - res.send('filestore sharelatex up') - } else { - logger.log('app is not ok - shutting down') - res.status(500).send('server is being shut down') - } + res.send('filestore sharelatex up') }) app.get('/health_check', healthCheckController.check) -app.get('*', (req, res) => res.sendStatus(404)) - const port = settings.internal.filestore.port || 3009 const host = '0.0.0.0' if (!module.parent) { // Called directly - const server = app.listen(port, host, error => { + app.listen(port, host, error => { if (error) { logger.error('Error starting Filestore', error) throw error } logger.info(`Filestore starting up, listening on ${host}:${port}`) }) - exceptionHandler.server = server } module.exports = app - -process.on('SIGTERM', function() { - logger.log('filestore got SIGTERM, shutting down gracefully') - exceptionHandler.beginShutdown() -}) - -if (global.gc) { - const oneMinute = 60 * 1000 - const gcTimer = setInterval(function() { - global.gc() - logger.log(process.memoryUsage(), 'global.gc') - }, 3 * oneMinute) - gcTimer.unref() -} diff --git a/services/filestore/app/js/ExceptionHandler.js b/services/filestore/app/js/ExceptionHandler.js deleted file mode 100644 index d52c00bb2c..0000000000 --- a/services/filestore/app/js/ExceptionHandler.js +++ /dev/null @@ -1,97 +0,0 @@ -const Metrics = require('metrics-sharelatex') -const logger = require('logger-sharelatex') - -// TODO: domain has been deprecated for some time - do we need it and is there a better way? - -// eslint-disable-next-line node/no-deprecated-api -const domain = require('domain') - -const TWO_MINUTES = 120 * 1000 - -class ExceptionHandler { - constructor() { - this._appIsOk = true - } - - beginShutdown() { - if (this._appIsOk) { - this._appIsOk = false - - // hard-terminate this process if graceful shutdown fails - const killTimer = setTimeout(() => process.exit(1), TWO_MINUTES) - - if (typeof killTimer.unref === 'function') { - killTimer.unref() - } // prevent timer from keeping process alive - - this.server.close(function() { - logger.log('closed all connections') - Metrics.close() - if (typeof process.disconnect === 'function') { - process.disconnect() - } - }) - logger.log('server will stop accepting connections') - } - } - - addMiddleware(app) { - app.use(this.middleware.bind(this)) - } - - appIsOk() { - return this._appIsOk - } - - setNotOk() { - this._appIsOk = false - } - - middleware(req, res, next) { - const rescueLogger = require('logger-sharelatex') - const requestDomain = domain.create() - requestDomain.add(req) - requestDomain.add(res) - requestDomain.on('error', err => { - try { - // request a shutdown to prevent memory leaks - this.beginShutdown() - if (!res.headerSent) { - res.status(500).send('uncaught exception') - } - req = { - body: req.body, - headers: req.headers, - url: req.url, - key: req.key, - statusCode: req.statusCode - } - err = { - message: err.message, - stack: err.stack, - name: err.name, - type: err.type, - arguments: err.arguments - } - rescueLogger.err( - { err, req, res }, - 'uncaught exception thrown on request' - ) - } catch (exception) { - rescueLogger.err( - { err: exception }, - 'exception in request domain handler' - ) - } - }) - - if (!this._appIsOk) { - // when shutting down, close any HTTP keep-alive connections - res.set('Connection', 'close') - } - - requestDomain.run(next) - } -} - -module.exports = ExceptionHandler diff --git a/services/filestore/test/acceptance/js/FilestoreApp.js b/services/filestore/test/acceptance/js/FilestoreApp.js index 32ec7a5adf..718d53bcf8 100644 --- a/services/filestore/test/acceptance/js/FilestoreApp.js +++ b/services/filestore/test/acceptance/js/FilestoreApp.js @@ -44,7 +44,6 @@ class FilestoreApp { resolve() } ) - this.app.exceptionHandler.server = this.server }) if (Settings.filestore.backend === 's3') { From 74b480fc55bd5475cf4c7e39aa724f110967f708 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Fri, 3 Jan 2020 10:06:19 +0000 Subject: [PATCH 61/73] Post-decaf cleanup of app.js --- services/filestore/app.js | 34 ++++++- services/filestore/app/js/ExceptionHandler.js | 97 +++++++++++++++++++ .../test/acceptance/js/FilestoreApp.js | 1 + 3 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 services/filestore/app/js/ExceptionHandler.js diff --git a/services/filestore/app.js b/services/filestore/app.js index 232c5b24bc..6bf68f6a34 100644 --- a/services/filestore/app.js +++ b/services/filestore/app.js @@ -12,8 +12,11 @@ const fileController = require('./app/js/FileController') const bucketController = require('./app/js/BucketController') const keyBuilder = require('./app/js/KeyBuilder') const healthCheckController = require('./app/js/HealthCheckController') +const ExceptionHandler = require('./app/js/ExceptionHandler') +const exceptionHandler = new ExceptionHandler() const app = express() +app.exceptionHandler = exceptionHandler if (settings.sentry && settings.sentry.dsn) { logger.initializeErrorReporting(settings.sentry.dsn) @@ -31,6 +34,8 @@ app.use(function(req, res, next) { next() }) +exceptionHandler.addMiddleware(app) + Metrics.injectMetricsRoute(app) app.head( @@ -128,12 +133,24 @@ app.get('/heapdump', (req, res, next) => ) ) +app.post('/shutdown', function(req, res) { + exceptionHandler.setNotOk() + res.sendStatus(200) +}) + app.get('/status', function(req, res) { - res.send('filestore sharelatex up') + if (exceptionHandler.appIsOk()) { + res.send('filestore sharelatex up') + } else { + logger.log('app is not ok - shutting down') + res.send('server is being shut down').status(500) + } }) app.get('/health_check', healthCheckController.check) +app.get('*', (req, res) => res.sendStatus(404)) + const port = settings.internal.filestore.port || 3009 const host = '0.0.0.0' @@ -146,6 +163,21 @@ if (!module.parent) { } logger.info(`Filestore starting up, listening on ${host}:${port}`) }) + exceptionHandler.server = server } module.exports = app + +process.on('SIGTERM', function() { + logger.log('filestore got SIGTERM, shutting down gracefully') + exceptionHandler.beginShutdown() +}) + +if (global.gc) { + const oneMinute = 60 * 1000 + const gcTimer = setInterval(function() { + global.gc() + logger.log(process.memoryUsage(), 'global.gc') + }, 3 * oneMinute) + gcTimer.unref() +} diff --git a/services/filestore/app/js/ExceptionHandler.js b/services/filestore/app/js/ExceptionHandler.js new file mode 100644 index 0000000000..122e78805c --- /dev/null +++ b/services/filestore/app/js/ExceptionHandler.js @@ -0,0 +1,97 @@ +const Metrics = require('metrics-sharelatex') +const logger = require('logger-sharelatex') + +// TODO: domain has been deprecated for some time - do we need it and is there a better way? + +// eslint-disable-next-line node/no-deprecated-api +const domain = require('domain') + +const TWO_MINUTES = 120 * 1000 + +class ExceptionHandler { + constructor() { + this._appIsOk = true + } + + beginShutdown() { + if (this._appIsOk) { + this._appIsOk = false + + // hard-terminate this process if graceful shutdown fails + const killTimer = setTimeout(() => process.exit(1), TWO_MINUTES) + + if (typeof killTimer.unref === 'function') { + killTimer.unref() + } // prevent timer from keeping process alive + + this.server.close(function() { + logger.log('closed all connections') + Metrics.close() + if (typeof process.disconnect === 'function') { + process.disconnect() + } + }) + logger.log('server will stop accepting connections') + } + } + + addMiddleware(app) { + app.use(this.middleware.bind(this)) + } + + appIsOk() { + return this._appIsOk + } + + setNotOk() { + this._appIsOk = false + } + + middleware(req, res, next) { + const rescueLogger = require('logger-sharelatex') + const requestDomain = domain.create() + requestDomain.add(req) + requestDomain.add(res) + requestDomain.on('error', err => { + try { + // request a shutdown to prevent memory leaks + this.beginShutdown() + if (!res.headerSent) { + res.send('uncaught exception').status(500) + } + req = { + body: req.body, + headers: req.headers, + url: req.url, + key: req.key, + statusCode: req.statusCode + } + err = { + message: err.message, + stack: err.stack, + name: err.name, + type: err.type, + arguments: err.arguments + } + rescueLogger.err( + { err, req, res }, + 'uncaught exception thrown on request' + ) + } catch (exception) { + rescueLogger.err( + { err: exception }, + 'exception in request domain handler' + ) + } + }) + + if (!this._appIsOk) { + // when shutting down, close any HTTP keep-alive connections + res.set('Connection', 'close') + } + + requestDomain.run(next) + } +} + +module.exports = ExceptionHandler diff --git a/services/filestore/test/acceptance/js/FilestoreApp.js b/services/filestore/test/acceptance/js/FilestoreApp.js index 718d53bcf8..32ec7a5adf 100644 --- a/services/filestore/test/acceptance/js/FilestoreApp.js +++ b/services/filestore/test/acceptance/js/FilestoreApp.js @@ -44,6 +44,7 @@ class FilestoreApp { resolve() } ) + this.app.exceptionHandler.server = this.server }) if (Settings.filestore.backend === 's3') { From 87b51d0c3516e9694dd28c97a9a0cebdfa10c5b5 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 6 Jan 2020 15:43:24 +0000 Subject: [PATCH 62/73] Remove old exception-handling and shutdown-related mechanisms --- services/filestore/app.js | 34 +------ services/filestore/app/js/ExceptionHandler.js | 97 ------------------- .../test/acceptance/js/FilestoreApp.js | 1 - 3 files changed, 1 insertion(+), 131 deletions(-) delete mode 100644 services/filestore/app/js/ExceptionHandler.js diff --git a/services/filestore/app.js b/services/filestore/app.js index 6bf68f6a34..232c5b24bc 100644 --- a/services/filestore/app.js +++ b/services/filestore/app.js @@ -12,11 +12,8 @@ const fileController = require('./app/js/FileController') const bucketController = require('./app/js/BucketController') const keyBuilder = require('./app/js/KeyBuilder') const healthCheckController = require('./app/js/HealthCheckController') -const ExceptionHandler = require('./app/js/ExceptionHandler') -const exceptionHandler = new ExceptionHandler() const app = express() -app.exceptionHandler = exceptionHandler if (settings.sentry && settings.sentry.dsn) { logger.initializeErrorReporting(settings.sentry.dsn) @@ -34,8 +31,6 @@ app.use(function(req, res, next) { next() }) -exceptionHandler.addMiddleware(app) - Metrics.injectMetricsRoute(app) app.head( @@ -133,24 +128,12 @@ app.get('/heapdump', (req, res, next) => ) ) -app.post('/shutdown', function(req, res) { - exceptionHandler.setNotOk() - res.sendStatus(200) -}) - app.get('/status', function(req, res) { - if (exceptionHandler.appIsOk()) { - res.send('filestore sharelatex up') - } else { - logger.log('app is not ok - shutting down') - res.send('server is being shut down').status(500) - } + res.send('filestore sharelatex up') }) app.get('/health_check', healthCheckController.check) -app.get('*', (req, res) => res.sendStatus(404)) - const port = settings.internal.filestore.port || 3009 const host = '0.0.0.0' @@ -163,21 +146,6 @@ if (!module.parent) { } logger.info(`Filestore starting up, listening on ${host}:${port}`) }) - exceptionHandler.server = server } module.exports = app - -process.on('SIGTERM', function() { - logger.log('filestore got SIGTERM, shutting down gracefully') - exceptionHandler.beginShutdown() -}) - -if (global.gc) { - const oneMinute = 60 * 1000 - const gcTimer = setInterval(function() { - global.gc() - logger.log(process.memoryUsage(), 'global.gc') - }, 3 * oneMinute) - gcTimer.unref() -} diff --git a/services/filestore/app/js/ExceptionHandler.js b/services/filestore/app/js/ExceptionHandler.js deleted file mode 100644 index 122e78805c..0000000000 --- a/services/filestore/app/js/ExceptionHandler.js +++ /dev/null @@ -1,97 +0,0 @@ -const Metrics = require('metrics-sharelatex') -const logger = require('logger-sharelatex') - -// TODO: domain has been deprecated for some time - do we need it and is there a better way? - -// eslint-disable-next-line node/no-deprecated-api -const domain = require('domain') - -const TWO_MINUTES = 120 * 1000 - -class ExceptionHandler { - constructor() { - this._appIsOk = true - } - - beginShutdown() { - if (this._appIsOk) { - this._appIsOk = false - - // hard-terminate this process if graceful shutdown fails - const killTimer = setTimeout(() => process.exit(1), TWO_MINUTES) - - if (typeof killTimer.unref === 'function') { - killTimer.unref() - } // prevent timer from keeping process alive - - this.server.close(function() { - logger.log('closed all connections') - Metrics.close() - if (typeof process.disconnect === 'function') { - process.disconnect() - } - }) - logger.log('server will stop accepting connections') - } - } - - addMiddleware(app) { - app.use(this.middleware.bind(this)) - } - - appIsOk() { - return this._appIsOk - } - - setNotOk() { - this._appIsOk = false - } - - middleware(req, res, next) { - const rescueLogger = require('logger-sharelatex') - const requestDomain = domain.create() - requestDomain.add(req) - requestDomain.add(res) - requestDomain.on('error', err => { - try { - // request a shutdown to prevent memory leaks - this.beginShutdown() - if (!res.headerSent) { - res.send('uncaught exception').status(500) - } - req = { - body: req.body, - headers: req.headers, - url: req.url, - key: req.key, - statusCode: req.statusCode - } - err = { - message: err.message, - stack: err.stack, - name: err.name, - type: err.type, - arguments: err.arguments - } - rescueLogger.err( - { err, req, res }, - 'uncaught exception thrown on request' - ) - } catch (exception) { - rescueLogger.err( - { err: exception }, - 'exception in request domain handler' - ) - } - }) - - if (!this._appIsOk) { - // when shutting down, close any HTTP keep-alive connections - res.set('Connection', 'close') - } - - requestDomain.run(next) - } -} - -module.exports = ExceptionHandler diff --git a/services/filestore/test/acceptance/js/FilestoreApp.js b/services/filestore/test/acceptance/js/FilestoreApp.js index 32ec7a5adf..718d53bcf8 100644 --- a/services/filestore/test/acceptance/js/FilestoreApp.js +++ b/services/filestore/test/acceptance/js/FilestoreApp.js @@ -44,7 +44,6 @@ class FilestoreApp { resolve() } ) - this.app.exceptionHandler.server = this.server }) if (Settings.filestore.backend === 's3') { From e27cf4db7bcb846505935eae612cc80550919ec6 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Fri, 3 Jan 2020 17:42:28 +0000 Subject: [PATCH 63/73] Pass pipeline errors onto 'next' error handler --- services/filestore/app/js/FileController.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/filestore/app/js/FileController.js b/services/filestore/app/js/FileController.js index dbba9a93cc..726438a557 100644 --- a/services/filestore/app/js/FileController.js +++ b/services/filestore/app/js/FileController.js @@ -17,7 +17,7 @@ module.exports = { directorySize } -function getFile(req, res) { +function getFile(req, res, next) { const { key, bucket } = req const { format, style } = req.query const options = { @@ -61,7 +61,8 @@ function getFile(req, res) { } logger.log({ key, bucket, format, style }, 'sending file to response') - pipeline(fileStream, res) + + pipeline(fileStream, res, next) }) } From 473aea4e60437096dffe77750413563a0fd030fa Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Fri, 3 Jan 2020 18:21:21 +0000 Subject: [PATCH 64/73] Remove AWSSDKPersistorManager --- .../app/js/AWSSDKPersistorManager.js | 197 ------- .../unit/js/AWSSDKPersistorManagerTests.js | 509 ------------------ 2 files changed, 706 deletions(-) delete mode 100644 services/filestore/app/js/AWSSDKPersistorManager.js delete mode 100644 services/filestore/test/unit/js/AWSSDKPersistorManagerTests.js diff --git a/services/filestore/app/js/AWSSDKPersistorManager.js b/services/filestore/app/js/AWSSDKPersistorManager.js deleted file mode 100644 index 4dbc836280..0000000000 --- a/services/filestore/app/js/AWSSDKPersistorManager.js +++ /dev/null @@ -1,197 +0,0 @@ -/* eslint-disable - handle-callback-err, - no-return-assign, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -// This module is not used in production, which currently uses -// S3PersistorManager. The intention is to migrate S3PersistorManager to use the -// latest aws-sdk and delete this module so that PersistorManager would load the -// same backend for both the 's3' and 'aws-sdk' options. - -const logger = require('logger-sharelatex') -const aws = require('aws-sdk') -const _ = require('underscore') -const fs = require('fs') -const Errors = require('./Errors') - -const s3 = new aws.S3() - -module.exports = { - sendFile(bucketName, key, fsPath, callback) { - logger.log({ bucketName, key }, 'send file data to s3') - const stream = fs.createReadStream(fsPath) - return s3.upload({ Bucket: bucketName, Key: key, Body: stream }, function( - err, - data - ) { - if (err != null) { - logger.err( - { err, Bucket: bucketName, Key: key }, - 'error sending file data to s3' - ) - } - return callback(err) - }) - }, - - sendStream(bucketName, key, stream, callback) { - logger.log({ bucketName, key }, 'send file stream to s3') - return s3.upload({ Bucket: bucketName, Key: key, Body: stream }, function( - err, - data - ) { - if (err != null) { - logger.err( - { err, Bucket: bucketName, Key: key }, - 'error sending file stream to s3' - ) - } - return callback(err) - }) - }, - - getFileStream(bucketName, key, opts, callback) { - if (callback == null) { - callback = function(err, res) {} - } - logger.log({ bucketName, key }, 'get file stream from s3') - callback = _.once(callback) - const params = { - Bucket: bucketName, - Key: key - } - if (opts.start != null && opts.end != null) { - params.Range = `bytes=${opts.start}-${opts.end}` - } - const request = s3.getObject(params) - const stream = request.createReadStream() - stream.on('readable', () => callback(null, stream)) - return stream.on('error', function(err) { - logger.err({ err, bucketName, key }, 'error getting file stream from s3') - if (err.code === 'NoSuchKey') { - return callback( - new Errors.NotFoundError(`File not found in S3: ${bucketName}:${key}`) - ) - } - return callback(err) - }) - }, - - copyFile(bucketName, sourceKey, destKey, callback) { - logger.log({ bucketName, sourceKey, destKey }, 'copying file in s3') - const source = bucketName + '/' + sourceKey - return s3.copyObject( - { Bucket: bucketName, Key: destKey, CopySource: source }, - function(err) { - if (err != null) { - logger.err( - { err, bucketName, sourceKey, destKey }, - 'something went wrong copying file in s3' - ) - } - return callback(err) - } - ) - }, - - deleteFile(bucketName, key, callback) { - logger.log({ bucketName, key }, 'delete file in s3') - return s3.deleteObject({ Bucket: bucketName, Key: key }, function(err) { - if (err != null) { - logger.err( - { err, bucketName, key }, - 'something went wrong deleting file in s3' - ) - } - return callback(err) - }) - }, - - deleteDirectory(bucketName, key, callback) { - logger.log({ bucketName, key }, 'delete directory in s3') - return s3.listObjects({ Bucket: bucketName, Prefix: key }, function( - err, - data - ) { - if (err != null) { - logger.err( - { err, bucketName, key }, - 'something went wrong listing prefix in s3' - ) - return callback(err) - } - if (data.Contents.length === 0) { - logger.log({ bucketName, key }, 'the directory is empty') - return callback() - } - const keys = _.map(data.Contents, entry => ({ - Key: entry.Key - })) - return s3.deleteObjects( - { - Bucket: bucketName, - Delete: { - Objects: keys, - Quiet: true - } - }, - function(err) { - if (err != null) { - logger.err( - { err, bucketName, key: keys }, - 'something went wrong deleting directory in s3' - ) - } - return callback(err) - } - ) - }) - }, - - checkIfFileExists(bucketName, key, callback) { - logger.log({ bucketName, key }, 'check file existence in s3') - return s3.headObject({ Bucket: bucketName, Key: key }, function(err, data) { - if (err != null) { - if (err.code === 'NotFound') { - return callback(null, false) - } - logger.err( - { err, bucketName, key }, - 'something went wrong checking head in s3' - ) - return callback(err) - } - return callback(null, data.ETag != null) - }) - }, - - directorySize(bucketName, key, callback) { - logger.log({ bucketName, key }, 'get project size in s3') - return s3.listObjects({ Bucket: bucketName, Prefix: key }, function( - err, - data - ) { - if (err != null) { - logger.err( - { err, bucketName, key }, - 'something went wrong listing prefix in s3' - ) - return callback(err) - } - if (data.Contents.length === 0) { - logger.log({ bucketName, key }, 'the directory is empty') - return callback() - } - let totalSize = 0 - _.each(data.Contents, entry => (totalSize += entry.Size)) - return callback(null, totalSize) - }) - } -} diff --git a/services/filestore/test/unit/js/AWSSDKPersistorManagerTests.js b/services/filestore/test/unit/js/AWSSDKPersistorManagerTests.js deleted file mode 100644 index ea88da71c3..0000000000 --- a/services/filestore/test/unit/js/AWSSDKPersistorManagerTests.js +++ /dev/null @@ -1,509 +0,0 @@ -/* eslint-disable - handle-callback-err, - no-dupe-keys, - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const sinon = require('sinon') -const chai = require('chai') - -const should = chai.should() -const { expect } = chai - -const modulePath = '../../../app/js/AWSSDKPersistorManager.js' -const SandboxedModule = require('sandboxed-module') - -describe('AWSSDKPersistorManager', function() { - beforeEach(function() { - this.settings = { - filestore: { - backend: 'aws-sdk' - } - } - this.s3 = { - upload: sinon.stub(), - getObject: sinon.stub(), - copyObject: sinon.stub(), - deleteObject: sinon.stub(), - listObjects: sinon.stub(), - deleteObjects: sinon.stub(), - headObject: sinon.stub() - } - this.awssdk = { S3: sinon.stub().returns(this.s3) } - - this.requires = { - 'aws-sdk': this.awssdk, - 'settings-sharelatex': this.settings, - 'logger-sharelatex': { - log() {}, - err() {} - }, - fs: (this.fs = { createReadStream: sinon.stub() }), - './Errors': (this.Errors = { NotFoundError: sinon.stub() }) - } - this.key = 'my/key' - this.bucketName = 'my-bucket' - this.error = 'my error' - return (this.AWSSDKPersistorManager = SandboxedModule.require(modulePath, { - requires: this.requires - })) - }) - - describe('sendFile', function() { - beforeEach(function() { - this.stream = {} - this.fsPath = '/usr/local/some/file' - return this.fs.createReadStream.returns(this.stream) - }) - - it('should put the file with s3.upload', function(done) { - this.s3.upload.callsArgWith(1) - return this.AWSSDKPersistorManager.sendFile( - this.bucketName, - this.key, - this.fsPath, - err => { - expect(err).to.not.be.ok - expect(this.s3.upload.calledOnce, 'called only once').to.be.true - expect( - this.s3.upload.calledWith({ - Bucket: this.bucketName, - Key: this.key, - Body: this.stream - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - return it('should dispatch the error from s3.upload', function(done) { - this.s3.upload.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.sendFile( - this.bucketName, - this.key, - this.fsPath, - err => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - }) - - describe('sendStream', function() { - beforeEach(function() { - return (this.stream = {}) - }) - - it('should put the file with s3.upload', function(done) { - this.s3.upload.callsArgWith(1) - return this.AWSSDKPersistorManager.sendStream( - this.bucketName, - this.key, - this.stream, - err => { - expect(err).to.not.be.ok - expect(this.s3.upload.calledOnce, 'called only once').to.be.true - expect( - this.s3.upload.calledWith({ - Bucket: this.bucketName, - Key: this.key, - Body: this.stream - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - return it('should dispatch the error from s3.upload', function(done) { - this.s3.upload.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.sendStream( - this.bucketName, - this.key, - this.stream, - err => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - }) - - describe('getFileStream', function() { - beforeEach(function() { - this.opts = {} - this.stream = {} - this.read_stream = { on: (this.read_stream_on = sinon.stub()) } - this.object = { createReadStream: sinon.stub().returns(this.read_stream) } - return this.s3.getObject.returns(this.object) - }) - - it('should return a stream from s3.getObject', function(done) { - this.read_stream_on.withArgs('readable').callsArgWith(1) - - return this.AWSSDKPersistorManager.getFileStream( - this.bucketName, - this.key, - this.opts, - (err, stream) => { - expect(this.read_stream_on.calledTwice) - expect(err).to.not.be.ok - expect(stream, 'returned the stream').to.equal(this.read_stream) - expect( - this.s3.getObject.calledWith({ - Bucket: this.bucketName, - Key: this.key - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - describe('with start and end options', function() { - beforeEach(function() { - return (this.opts = { - start: 0, - end: 8 - }) - }) - return it('should pass headers to the s3.GetObject', function(done) { - this.read_stream_on.withArgs('readable').callsArgWith(1) - this.AWSSDKPersistorManager.getFileStream( - this.bucketName, - this.key, - this.opts, - (err, stream) => { - return expect( - this.s3.getObject.calledWith({ - Bucket: this.bucketName, - Key: this.key, - Range: 'bytes=0-8' - }), - 'called with correct arguments' - ).to.be.true - } - ) - return done() - }) - }) - - return describe('error conditions', function() { - describe("when the file doesn't exist", function() { - beforeEach(function() { - this.error = new Error() - return (this.error.code = 'NoSuchKey') - }) - return it('should produce a NotFoundError', function(done) { - this.read_stream_on.withArgs('error').callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.getFileStream( - this.bucketName, - this.key, - this.opts, - (err, stream) => { - expect(stream).to.not.be.ok - expect(err).to.be.ok - expect( - err instanceof this.Errors.NotFoundError, - 'error is a correct instance' - ).to.equal(true) - return done() - } - ) - }) - }) - - return describe('when there is some other error', function() { - beforeEach(function() { - return (this.error = new Error()) - }) - return it('should dispatch the error from s3 object stream', function(done) { - this.read_stream_on.withArgs('error').callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.getFileStream( - this.bucketName, - this.key, - this.opts, - (err, stream) => { - expect(stream).to.not.be.ok - expect(err).to.be.ok - expect(err).to.equal(this.error) - return done() - } - ) - }) - }) - }) - }) - - describe('copyFile', function() { - beforeEach(function() { - this.destKey = 'some/key' - return (this.stream = {}) - }) - - it('should copy the file with s3.copyObject', function(done) { - this.s3.copyObject.callsArgWith(1) - return this.AWSSDKPersistorManager.copyFile( - this.bucketName, - this.key, - this.destKey, - err => { - expect(err).to.not.be.ok - expect(this.s3.copyObject.calledOnce, 'called only once').to.be.true - expect( - this.s3.copyObject.calledWith({ - Bucket: this.bucketName, - Key: this.destKey, - CopySource: this.bucketName + '/' + this.key - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - return it('should dispatch the error from s3.copyObject', function(done) { - this.s3.copyObject.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.copyFile( - this.bucketName, - this.key, - this.destKey, - err => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - }) - - describe('deleteFile', function() { - it('should delete the file with s3.deleteObject', function(done) { - this.s3.deleteObject.callsArgWith(1) - return this.AWSSDKPersistorManager.deleteFile( - this.bucketName, - this.key, - err => { - expect(err).to.not.be.ok - expect(this.s3.deleteObject.calledOnce, 'called only once').to.be.true - expect( - this.s3.deleteObject.calledWith({ - Bucket: this.bucketName, - Key: this.key - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - return it('should dispatch the error from s3.deleteObject', function(done) { - this.s3.deleteObject.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.deleteFile( - this.bucketName, - this.key, - err => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - }) - - describe('deleteDirectory', function() { - it('should list the directory content using s3.listObjects', function(done) { - this.s3.listObjects.callsArgWith(1, null, { Contents: [] }) - return this.AWSSDKPersistorManager.deleteDirectory( - this.bucketName, - this.key, - err => { - expect(err).to.not.be.ok - expect(this.s3.listObjects.calledOnce, 'called only once').to.be.true - expect( - this.s3.listObjects.calledWith({ - Bucket: this.bucketName, - Prefix: this.key - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - it('should dispatch the error from s3.listObjects', function(done) { - this.s3.listObjects.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.deleteDirectory( - this.bucketName, - this.key, - err => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - - return describe('with directory content', function() { - beforeEach(function() { - return (this.fileList = [{ Key: 'foo' }, { Key: 'bar', Key: 'baz' }]) - }) - - it('should forward the file keys to s3.deleteObjects', function(done) { - this.s3.listObjects.callsArgWith(1, null, { Contents: this.fileList }) - this.s3.deleteObjects.callsArgWith(1) - return this.AWSSDKPersistorManager.deleteDirectory( - this.bucketName, - this.key, - err => { - expect(err).to.not.be.ok - expect(this.s3.deleteObjects.calledOnce, 'called only once').to.be - .true - expect( - this.s3.deleteObjects.calledWith({ - Bucket: this.bucketName, - Delete: { - Quiet: true, - Objects: this.fileList - } - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - return it('should dispatch the error from s3.deleteObjects', function(done) { - this.s3.listObjects.callsArgWith(1, null, { Contents: this.fileList }) - this.s3.deleteObjects.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.deleteDirectory( - this.bucketName, - this.key, - err => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - }) - }) - - describe('checkIfFileExists', function() { - it('should check for the file with s3.headObject', function(done) { - this.s3.headObject.callsArgWith(1, null, {}) - return this.AWSSDKPersistorManager.checkIfFileExists( - this.bucketName, - this.key, - (err, exists) => { - expect(err).to.not.be.ok - expect(this.s3.headObject.calledOnce, 'called only once').to.be.true - expect( - this.s3.headObject.calledWith({ - Bucket: this.bucketName, - Key: this.key - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - it('should return false on an inexistant file', function(done) { - this.s3.headObject.callsArgWith(1, null, {}) - return this.AWSSDKPersistorManager.checkIfFileExists( - this.bucketName, - this.key, - (err, exists) => { - expect(exists).to.be.false - return done() - } - ) - }) - - it('should return true on an existing file', function(done) { - this.s3.headObject.callsArgWith(1, null, { ETag: 'etag' }) - return this.AWSSDKPersistorManager.checkIfFileExists( - this.bucketName, - this.key, - (err, exists) => { - expect(exists).to.be.true - return done() - } - ) - }) - - return it('should dispatch the error from s3.headObject', function(done) { - this.s3.headObject.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.checkIfFileExists( - this.bucketName, - this.key, - (err, exists) => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - }) - - return describe('directorySize', function() { - it('should list the directory content using s3.listObjects', function(done) { - this.s3.listObjects.callsArgWith(1, null, { Contents: [] }) - return this.AWSSDKPersistorManager.directorySize( - this.bucketName, - this.key, - err => { - expect(err).to.not.be.ok - expect(this.s3.listObjects.calledOnce, 'called only once').to.be.true - expect( - this.s3.listObjects.calledWith({ - Bucket: this.bucketName, - Prefix: this.key - }), - 'called with correct arguments' - ).to.be.true - return done() - } - ) - }) - - it('should dispatch the error from s3.listObjects', function(done) { - this.s3.listObjects.callsArgWith(1, this.error) - return this.AWSSDKPersistorManager.directorySize( - this.bucketName, - this.key, - err => { - expect(err).to.equal(this.error) - return done() - } - ) - }) - - return it('should sum directory files sizes', function(done) { - this.s3.listObjects.callsArgWith(1, null, { - Contents: [{ Size: 1024 }, { Size: 2048 }] - }) - return this.AWSSDKPersistorManager.directorySize( - this.bucketName, - this.key, - (err, size) => { - expect(size).to.equal(3072) - return done() - } - ) - }) - }) -}) From 2ca74fdf15563092db8d2a1c45fe9717dbb45df1 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Fri, 3 Jan 2020 18:22:08 +0000 Subject: [PATCH 65/73] Cleanup and refactor S3PersistorManager to use aws-sdk only --- .../filestore/app/js/S3PersistorManager.js | 588 ++++----- services/filestore/npm-shrinkwrap.json | 34 +- services/filestore/package.json | 1 + .../test/unit/js/S3PersistorManagerTests.js | 1152 +++++++++-------- 4 files changed, 896 insertions(+), 879 deletions(-) diff --git a/services/filestore/app/js/S3PersistorManager.js b/services/filestore/app/js/S3PersistorManager.js index cadf38172a..5f8bb34c5d 100644 --- a/services/filestore/app/js/S3PersistorManager.js +++ b/services/filestore/app/js/S3PersistorManager.js @@ -1,376 +1,258 @@ -/* eslint-disable - handle-callback-err, - new-cap, - no-return-assign, - no-unused-vars, - node/no-deprecated-api, - standard/no-callback-literal, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -// This module is the one which is used in production. It needs to be migrated -// to use aws-sdk throughout, see the comments in AWSSDKPersistorManager for -// details. The knox library is unmaintained and has bugs. - const http = require('http') -http.globalAgent.maxSockets = 300 const https = require('https') +http.globalAgent.maxSockets = 300 https.globalAgent.maxSockets = 300 + const settings = require('settings-sharelatex') -const request = require('request') const logger = require('logger-sharelatex') const metrics = require('metrics-sharelatex') + +const meter = require('stream-meter') const fs = require('fs') -const knox = require('knox') -const path = require('path') -const LocalFileWriter = require('./LocalFileWriter') -const Errors = require('./Errors') -const _ = require('underscore') -const awsS3 = require('aws-sdk/clients/s3') -const URL = require('url') +const S3 = require('aws-sdk/clients/s3') +const { URL } = require('url') +const { callbackify } = require('util') +const { WriteError, ReadError, NotFoundError } = require('./Errors') -const thirtySeconds = 30 * 1000 - -const buildDefaultOptions = function(bucketName, method, key) { - let endpoint - if (settings.filestore.s3.endpoint) { - endpoint = `${settings.filestore.s3.endpoint}/${bucketName}` - } else { - endpoint = `https://${bucketName}.s3.amazonaws.com` - } - return { - aws: { - key: settings.filestore.s3.key, - secret: settings.filestore.s3.secret, - bucket: bucketName - }, - method, - timeout: thirtySeconds, - uri: `${endpoint}/${key}` +module.exports = { + sendFile: callbackify(sendFile), + sendStream: callbackify(sendStream), + getFileStream: callbackify(getFileStream), + deleteDirectory: callbackify(deleteDirectory), + getFileSize: callbackify(getFileSize), + deleteFile: callbackify(deleteFile), + copyFile: callbackify(copyFile), + checkIfFileExists: callbackify(checkIfFileExists), + getDirectorySize: callbackify(getDirectorySize), + promises: { + sendFile, + sendStream, + getFileStream, + deleteDirectory, + getFileSize, + deleteFile, + copyFile, + checkIfFileExists, + getDirectorySize } } -const getS3Options = function(credentials) { +const _client = new S3(_defaultOptions()) + +async function sendFile(bucketName, key, fsPath) { + let readStream + try { + readStream = fs.createReadStream(fsPath) + } catch (err) { + throw _wrapError( + err, + 'error reading file from disk', + { bucketName, key, fsPath }, + ReadError + ) + } + return sendStream(bucketName, key, readStream) +} + +async function sendStream(bucketName, key, readStream) { + try { + const meteredStream = meter() + meteredStream.on('finish', () => { + metrics.count('s3.egress', meteredStream.bytes) + }) + + const response = await _client + .upload({ + Bucket: bucketName, + Key: key, + Body: readStream.pipe(meteredStream) + }) + .promise() + + logger.log({ response, bucketName, key }, 'data uploaded to s3') + } catch (err) { + throw _wrapError( + err, + 'upload to S3 failed', + { bucketName, key }, + WriteError + ) + } +} + +async function getFileStream(bucketName, key, opts) { + opts = opts || {} + + const params = { + Bucket: bucketName, + Key: key + } + if (opts.start != null && opts.end != null) { + params.Range = `bytes=${opts.start}-${opts.end}` + } + + return new Promise((resolve, reject) => { + const stream = _client.getObject(params).createReadStream() + + const meteredStream = meter() + meteredStream.on('finish', () => { + metrics.count('s3.ingress', meteredStream.bytes) + }) + + const onStreamReady = function() { + stream.removeListener('readable', onStreamReady) + resolve(stream.pipe(meteredStream)) + } + stream.on('readable', onStreamReady) + stream.on('error', err => { + reject(_wrapError(err, 'error reading from S3', params, ReadError)) + }) + }) +} + +async function deleteDirectory(bucketName, key) { + logger.log({ key, bucketName }, 'deleting directory') + let response + + try { + response = await _client + .listObjects({ Bucket: bucketName, Prefix: key }) + .promise() + } catch (err) { + throw _wrapError( + err, + 'failed to list objects in S3', + { bucketName, key }, + ReadError + ) + } + + const objects = response.Contents.map(item => ({ Key: item.Key })) + if (objects.length) { + try { + await _client + .deleteObjects({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: true + } + }) + .promise() + } catch (err) { + throw _wrapError( + err, + 'failed to delete objects in S3', + { bucketName, key }, + WriteError + ) + } + } +} + +async function getFileSize(bucketName, key) { + try { + const response = await _client + .headObject({ Bucket: bucketName, Key: key }) + .promise() + return response.ContentLength + } catch (err) { + throw _wrapError( + err, + 'error getting size of s3 object', + { bucketName, key }, + ReadError + ) + } +} + +async function deleteFile(bucketName, key) { + try { + await _client.deleteObject({ Bucket: bucketName, Key: key }).promise() + } catch (err) { + throw _wrapError( + err, + 'failed to delete file in S3', + { bucketName, key }, + WriteError + ) + } +} + +async function copyFile(bucketName, sourceKey, destKey) { + const params = { + Bucket: bucketName, + Key: destKey, + CopySource: `${bucketName}/${sourceKey}` + } + try { + await _client.copyObject(params).promise() + } catch (err) { + throw _wrapError(err, 'failed to copy file in S3', params, WriteError) + } +} + +async function checkIfFileExists(bucketName, key) { + try { + await getFileSize(bucketName, key) + return true + } catch (err) { + if (err instanceof NotFoundError) { + return false + } + throw _wrapError( + err, + 'error checking whether S3 object exists', + { bucketName, key }, + ReadError + ) + } +} + +async function getDirectorySize(bucketName, key) { + try { + const response = await _client + .listObjects({ Bucket: bucketName, Prefix: key }) + .promise() + + return response.Contents.reduce((acc, item) => item.Size + acc, 0) + } catch (err) { + throw _wrapError( + err, + 'error getting directory size in S3', + { bucketName, key }, + ReadError + ) + } +} + +function _wrapError(error, message, params, ErrorType) { + if (['NoSuchKey', 'NotFound', 'ENOENT'].includes(error.code)) { + return new NotFoundError({ + message: 'no such file', + info: params + }).withCause(error) + } else { + return new ErrorType({ + message: message, + info: params + }).withCause(error) + } +} + +function _defaultOptions() { const options = { credentials: { - accessKeyId: credentials.auth_key, - secretAccessKey: credentials.auth_secret + accessKeyId: settings.filestore.s3.key, + secretAccessKey: settings.filestore.s3.secret } } if (settings.filestore.s3.endpoint) { - const endpoint = URL.parse(settings.filestore.s3.endpoint) + const endpoint = new URL(settings.filestore.s3.endpoint) options.endpoint = settings.filestore.s3.endpoint options.sslEnabled = endpoint.protocol === 'https' } return options } - -const defaultS3Client = new awsS3( - getS3Options({ - auth_key: settings.filestore.s3.key, - auth_secret: settings.filestore.s3.secret - }) -) - -const getS3Client = function(credentials) { - if (credentials != null) { - return new awsS3(getS3Options(credentials)) - } else { - return defaultS3Client - } -} - -const getKnoxClient = bucketName => { - const options = { - key: settings.filestore.s3.key, - secret: settings.filestore.s3.secret, - bucket: bucketName - } - if (settings.filestore.s3.endpoint) { - const endpoint = URL.parse(settings.filestore.s3.endpoint) - options.endpoint = endpoint.hostname - options.port = endpoint.port - } - return knox.createClient(options) -} - -module.exports = { - sendFile(bucketName, key, fsPath, callback) { - const s3Client = getKnoxClient(bucketName) - let uploaded = 0 - const putEventEmiter = s3Client.putFile(fsPath, key, function(err, res) { - metrics.count('s3.egress', uploaded) - if (err != null) { - logger.err( - { err, bucketName, key, fsPath }, - 'something went wrong uploading file to s3' - ) - return callback(err) - } - if (res == null) { - logger.err( - { err, res, bucketName, key, fsPath }, - 'no response from s3 put file' - ) - return callback('no response from put file') - } - if (res.statusCode !== 200) { - logger.err( - { bucketName, key, fsPath }, - 'non 200 response from s3 putting file' - ) - return callback('non 200 response from s3 on put file') - } - logger.log({ res, bucketName, key, fsPath }, 'file uploaded to s3') - return callback(err) - }) - putEventEmiter.on('error', function(err) { - logger.err( - { err, bucketName, key, fsPath }, - 'error emmited on put of file' - ) - return callback(err) - }) - return putEventEmiter.on( - 'progress', - progress => (uploaded = progress.written) - ) - }, - - sendStream(bucketName, key, readStream, callback) { - logger.log({ bucketName, key }, 'sending file to s3') - readStream.on('error', err => - logger.err({ bucketName, key }, 'error on stream to send to s3') - ) - return LocalFileWriter.writeStream(readStream, null, (err, fsPath) => { - if (err != null) { - logger.err( - { bucketName, key, fsPath, err }, - 'something went wrong writing stream to disk' - ) - return callback(err) - } - return this.sendFile(bucketName, key, fsPath, ( - err // delete the temporary file created above and return the original error - ) => LocalFileWriter.deleteFile(fsPath, () => callback(err))) - }) - }, - - // opts may be {start: Number, end: Number} - getFileStream(bucketName, key, opts, callback) { - if (callback == null) { - callback = function(err, res) {} - } - opts = opts || {} - callback = _.once(callback) - logger.log({ bucketName, key }, 'getting file from s3') - - const s3 = getS3Client(opts.credentials) - const s3Params = { - Bucket: bucketName, - Key: key - } - if (opts.start != null && opts.end != null) { - s3Params.Range = `bytes=${opts.start}-${opts.end}` - } - const s3Request = s3.getObject(s3Params) - - s3Request.on( - 'httpHeaders', - (statusCode, headers, response, statusMessage) => { - if ([403, 404].includes(statusCode)) { - // S3 returns a 403 instead of a 404 when the user doesn't have - // permission to list the bucket contents. - logger.log({ bucketName, key }, 'file not found in s3') - return callback( - new Errors.NotFoundError( - `File not found in S3: ${bucketName}:${key}` - ), - null - ) - } - if (![200, 206].includes(statusCode)) { - logger.log( - { bucketName, key }, - `error getting file from s3: ${statusCode}` - ) - return callback( - new Error( - `Got non-200 response from S3: ${statusCode} ${statusMessage}` - ), - null - ) - } - const stream = response.httpResponse.createUnbufferedStream() - stream.on('data', data => metrics.count('s3.ingress', data.byteLength)) - - return callback(null, stream) - } - ) - - s3Request.on('error', err => { - logger.err({ err, bucketName, key }, 'error getting file stream from s3') - return callback(err) - }) - - return s3Request.send() - }, - - getFileSize(bucketName, key, callback) { - logger.log({ bucketName, key }, 'getting file size from S3') - const s3 = getS3Client() - return s3.headObject({ Bucket: bucketName, Key: key }, function(err, data) { - if (err != null) { - if ([403, 404].includes(err.statusCode)) { - // S3 returns a 403 instead of a 404 when the user doesn't have - // permission to list the bucket contents. - logger.log( - { - bucketName, - key - }, - 'file not found in s3' - ) - callback( - new Errors.NotFoundError( - `File not found in S3: ${bucketName}:${key}` - ) - ) - } else { - logger.err( - { - bucketName, - key, - err - }, - 'error performing S3 HeadObject' - ) - callback(err) - } - return - } - return callback(null, data.ContentLength) - }) - }, - - copyFile(bucketName, sourceKey, destKey, callback) { - logger.log({ bucketName, sourceKey, destKey }, 'copying file in s3') - const source = bucketName + '/' + sourceKey - // use the AWS SDK instead of knox due to problems with error handling (https://github.com/Automattic/knox/issues/114) - const s3 = getS3Client() - return s3.copyObject( - { Bucket: bucketName, Key: destKey, CopySource: source }, - function(err) { - if (err != null) { - if (err.code === 'NoSuchKey') { - logger.err( - { bucketName, sourceKey }, - 'original file not found in s3 when copying' - ) - return callback( - new Errors.NotFoundError( - 'original file not found in S3 when copying' - ) - ) - } else { - logger.err( - { err, bucketName, sourceKey, destKey }, - 'something went wrong copying file in aws' - ) - return callback(err) - } - } else { - return callback() - } - } - ) - }, - - deleteFile(bucketName, key, callback) { - logger.log({ bucketName, key }, 'delete file in s3') - const options = buildDefaultOptions(bucketName, 'delete', key) - return request(options, function(err, res) { - if (err != null) { - logger.err( - { err, res, bucketName, key }, - 'something went wrong deleting file in aws' - ) - } - return callback(err) - }) - }, - - deleteDirectory(bucketName, key, _callback) { - // deleteMultiple can call the callback multiple times so protect against this. - const callback = function(...args) { - _callback(...Array.from(args || [])) - return (_callback = function() {}) - } - - logger.log({ key, bucketName }, 'deleting directory') - const s3Client = getKnoxClient(bucketName) - return s3Client.list({ prefix: key }, function(err, data) { - if (err != null) { - logger.err( - { err, bucketName, key }, - 'something went wrong listing prefix in aws' - ) - return callback(err) - } - const keys = _.map(data.Contents, entry => entry.Key) - return s3Client.deleteMultiple(keys, callback) - }) - }, - - checkIfFileExists(bucketName, key, callback) { - logger.log({ bucketName, key }, 'checking if file exists in s3') - const options = buildDefaultOptions(bucketName, 'head', key) - return request(options, function(err, res) { - if (err != null) { - logger.err( - { err, res, bucketName, key }, - 'something went wrong checking file in aws' - ) - return callback(err) - } - if (res == null) { - logger.err( - { err, res, bucketName, key }, - 'no response object returned when checking if file exists' - ) - err = new Error(`no response from s3 ${bucketName} ${key}`) - return callback(err) - } - const exists = res.statusCode === 200 - logger.log({ bucketName, key, exists }, 'checked if file exsists in s3') - return callback(err, exists) - }) - }, - - directorySize(bucketName, key, callback) { - logger.log({ bucketName, key }, 'get project size in s3') - const s3Client = getKnoxClient(bucketName) - return s3Client.list({ prefix: key }, function(err, data) { - if (err != null) { - logger.err( - { err, bucketName, key }, - 'something went wrong listing prefix in aws' - ) - return callback(err) - } - let totalSize = 0 - _.each(data.Contents, entry => (totalSize += entry.Size)) - logger.log({ totalSize }, 'total size') - return callback(null, totalSize) - }) - } -} diff --git a/services/filestore/npm-shrinkwrap.json b/services/filestore/npm-shrinkwrap.json index 3ed1400a61..8d78271caa 100644 --- a/services/filestore/npm-shrinkwrap.json +++ b/services/filestore/npm-shrinkwrap.json @@ -5018,6 +5018,38 @@ "resolved": "https://registry.npmjs.org/stream-counter/-/stream-counter-1.0.0.tgz", "integrity": "sha1-kc8lac5NxQYf6816yyY5SloRR1E=" }, + "stream-meter": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/stream-meter/-/stream-meter-1.0.4.tgz", + "integrity": "sha1-Uq+Vql6nYKJJFxZwTb/5D3Ov3R0=", + "requires": { + "readable-stream": "^2.1.4" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "stream-shift": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.0.tgz", @@ -5531,7 +5563,7 @@ "xml2js": { "version": "0.4.19", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", - "integrity": "sha1-aGwg8hMgnpSr8NG88e+qKRx4J6c=", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", "requires": { "sax": ">=0.6.0", "xmlbuilder": "~9.0.1" diff --git a/services/filestore/package.json b/services/filestore/package.json index 5d7c3e3ec1..14e35cd8a2 100644 --- a/services/filestore/package.json +++ b/services/filestore/package.json @@ -44,6 +44,7 @@ "settings-sharelatex": "^1.1.0", "stream-browserify": "^2.0.1", "stream-buffers": "~0.2.5", + "stream-meter": "^1.0.4", "underscore": "~1.5.2" }, "devDependencies": { diff --git a/services/filestore/test/unit/js/S3PersistorManagerTests.js b/services/filestore/test/unit/js/S3PersistorManagerTests.js index 777c9c5a32..3caf2db73d 100644 --- a/services/filestore/test/unit/js/S3PersistorManagerTests.js +++ b/services/filestore/test/unit/js/S3PersistorManagerTests.js @@ -1,618 +1,720 @@ -/* eslint-disable - handle-callback-err, - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const { assert } = require('chai') const sinon = require('sinon') const chai = require('chai') -const should = chai.should() const { expect } = chai const modulePath = '../../../app/js/S3PersistorManager.js' const SandboxedModule = require('sandboxed-module') +const Errors = require('../../../app/js/Errors') + describe('S3PersistorManagerTests', function() { - beforeEach(function() { - this.settings = { - filestore: { - backend: 's3', - s3: { - secret: 'secret', - key: 'this_key' - }, - stores: { - user_files: 'sl_user_files' - } - } - } - this.knoxClient = { - putFile: sinon.stub(), - copyFile: sinon.stub(), - list: sinon.stub(), - deleteMultiple: sinon.stub(), - get: sinon.stub() - } - this.knox = { createClient: sinon.stub().returns(this.knoxClient) } - this.s3EventHandlers = {} - this.s3Request = { - on: sinon.stub().callsFake((event, callback) => { - return (this.s3EventHandlers[event] = callback) - }), - send: sinon.stub() - } - this.s3Response = { - httpResponse: { - createUnbufferedStream: sinon.stub() - } - } - this.s3Client = { - copyObject: sinon.stub(), - headObject: sinon.stub(), - getObject: sinon.stub().returns(this.s3Request) - } - this.awsS3 = sinon.stub().returns(this.s3Client) - this.LocalFileWriter = { - writeStream: sinon.stub(), - deleteFile: sinon.stub() - } - this.request = sinon.stub() - this.requires = { - knox: this.knox, - 'aws-sdk/clients/s3': this.awsS3, - 'settings-sharelatex': this.settings, - './LocalFileWriter': this.LocalFileWriter, - 'logger-sharelatex': { - log() {}, - err() {} + const settings = { + filestore: { + backend: 's3', + s3: { + secret: 'secret', + key: 'this_key' }, - request: this.request, - './Errors': (this.Errors = { NotFoundError: sinon.stub() }) + stores: { + user_files: 'sl_user_files' + } } - this.key = 'my/key' - this.bucketName = 'my-bucket' - this.error = 'my errror' - return (this.S3PersistorManager = SandboxedModule.require(modulePath, { - requires: this.requires - })) + } + const filename = '/wombat/potato.tex' + const bucket = 'womBucket' + const key = 'monKey' + const destKey = 'donKey' + const objectSize = 5555 + const genericError = new Error('guru meditation error') + const files = [ + { Key: 'llama', Size: 11 }, + { Key: 'hippo', Size: 22 } + ] + const filesSize = 33 + + let Metrics, + S3, + Fs, + Meter, + MeteredStream, + ReadStream, + S3PersistorManager, + S3Client, + S3ReadStream, + S3NotFoundError, + FileNotFoundError, + EmptyPromise + + beforeEach(function() { + EmptyPromise = { + promise: sinon.stub().resolves() + } + + Metrics = { + count: sinon.stub() + } + + ReadStream = { + pipe: sinon.stub().returns('readStream') + } + + FileNotFoundError = new Error('File not found') + FileNotFoundError.code = 'ENOENT' + + Fs = { + createReadStream: sinon.stub().returns(ReadStream) + } + + MeteredStream = { + on: sinon.stub(), + bytes: objectSize + } + MeteredStream.on.withArgs('finish').yields() + Meter = sinon.stub().returns(MeteredStream) + + S3NotFoundError = new Error('not found') + S3NotFoundError.code = 'NoSuchKey' + + S3ReadStream = { + on: sinon.stub(), + pipe: sinon.stub().returns('s3Stream'), + removeListener: sinon.stub() + } + S3ReadStream.on.withArgs('readable').yields() + S3Client = { + getObject: sinon.stub().returns({ + createReadStream: sinon.stub().returns(S3ReadStream) + }), + headObject: sinon.stub().returns({ + promise: sinon.stub().resolves({ + ContentLength: objectSize + }) + }), + listObjects: sinon.stub().returns({ + promise: sinon.stub().resolves({ + Contents: files + }) + }), + upload: sinon.stub().returns(EmptyPromise), + copyObject: sinon.stub().returns(EmptyPromise), + deleteObject: sinon.stub().returns(EmptyPromise), + deleteObjects: sinon.stub().returns(EmptyPromise) + } + S3 = sinon.stub().returns(S3Client) + + S3PersistorManager = SandboxedModule.require(modulePath, { + requires: { + 'aws-sdk/clients/s3': S3, + 'settings-sharelatex': settings, + './Errors': Errors, + fs: Fs, + 'stream-meter': Meter, + 'logger-sharelatex': { + log() {}, + err() {} + }, + 'metrics-sharelatex': Metrics + }, + globals: { console } + }) }) describe('getFileStream', function() { - describe('success', function() { - beforeEach(function() { - this.expectedStream = { expectedStream: true } - this.expectedStream.on = sinon.stub() - this.s3Request.send.callsFake(() => { - return this.s3EventHandlers.httpHeaders( - 200, - {}, - this.s3Response, - 'OK' - ) - }) - return this.s3Response.httpResponse.createUnbufferedStream.returns( - this.expectedStream - ) + describe('when called with valid parameters', function() { + let stream + + beforeEach(async function() { + stream = await S3PersistorManager.promises.getFileStream(bucket, key) }) - it('returns a stream', function(done) { - return this.S3PersistorManager.getFileStream( - this.bucketName, - this.key, - {}, - (err, stream) => { - if (err != null) { - return done(err) - } - expect(stream).to.equal(this.expectedStream) - return done() - } - ) + it('returns a stream', function() { + expect(stream).to.equal('s3Stream') }) - it('sets the AWS client up with credentials from settings', function(done) { - return this.S3PersistorManager.getFileStream( - this.bucketName, - this.key, - {}, - (err, stream) => { - if (err != null) { - return done(err) - } - expect(this.awsS3.lastCall.args).to.deep.equal([ - { - credentials: { - accessKeyId: this.settings.filestore.s3.key, - secretAccessKey: this.settings.filestore.s3.secret - } - } - ]) - return done() - } - ) - }) - - it('fetches the right key from the right bucket', function(done) { - return this.S3PersistorManager.getFileStream( - this.bucketName, - this.key, - {}, - (err, stream) => { - if (err != null) { - return done(err) - } - expect(this.s3Client.getObject.lastCall.args).to.deep.equal([ - { - Bucket: this.bucketName, - Key: this.key - } - ]) - return done() - } - ) - }) - - it('accepts alternative credentials', function(done) { - const accessKeyId = 'that_key' - const secret = 'that_secret' - const opts = { + it('sets the AWS client up with credentials from settings', function() { + expect(S3).to.have.been.calledWith({ credentials: { - auth_key: accessKeyId, - auth_secret: secret + accessKeyId: settings.filestore.s3.key, + secretAccessKey: settings.filestore.s3.secret } - } - return this.S3PersistorManager.getFileStream( - this.bucketName, - this.key, - opts, - (err, stream) => { - if (err != null) { - return done(err) - } - expect(this.awsS3.lastCall.args).to.deep.equal([ - { - credentials: { - accessKeyId, - secretAccessKey: secret - } - } - ]) - expect(stream).to.equal(this.expectedStream) - return done() - } - ) + }) }) - return it('accepts byte range', function(done) { - const start = 0 - const end = 8 - const opts = { start, end } - return this.S3PersistorManager.getFileStream( - this.bucketName, - this.key, - opts, - (err, stream) => { - if (err != null) { - return done(err) - } - expect(this.s3Client.getObject.lastCall.args).to.deep.equal([ - { - Bucket: this.bucketName, - Key: this.key, - Range: `bytes=${start}-${end}` - } - ]) - expect(stream).to.equal(this.expectedStream) - return done() - } - ) + it('fetches the right key from the right bucket', function() { + expect(S3Client.getObject).to.have.been.calledWith({ + Bucket: bucket, + Key: key + }) + }) + + it('pipes the stream through the meter', function() { + expect(S3ReadStream.pipe).to.have.been.calledWith(MeteredStream) + }) + + it('records an ingress metric', function() { + expect(Metrics.count).to.have.been.calledWith('s3.ingress', objectSize) }) }) - return describe('errors', function() { - describe("when the file doesn't exist", function() { - beforeEach(function() { - return this.s3Request.send.callsFake(() => { - return this.s3EventHandlers.httpHeaders( - 404, - {}, - this.s3Response, - 'Not found' - ) - }) - }) + describe('when called with a byte range', function() { + let stream - return it('returns a NotFoundError that indicates the bucket and key', function(done) { - return this.S3PersistorManager.getFileStream( - this.bucketName, - this.key, - {}, - (err, stream) => { - expect(err).to.be.instanceof(this.Errors.NotFoundError) - const errMsg = this.Errors.NotFoundError.lastCall.args[0] - expect(errMsg).to.match(new RegExp(`.*${this.bucketName}.*`)) - expect(errMsg).to.match(new RegExp(`.*${this.key}.*`)) - return done() - } - ) + beforeEach(async function() { + stream = await S3PersistorManager.promises.getFileStream(bucket, key, { + start: 5, + end: 10 }) }) - describe('when S3 encounters an unkown error', function() { - beforeEach(function() { - return this.s3Request.send.callsFake(() => { - return this.s3EventHandlers.httpHeaders( - 500, - {}, - this.s3Response, - 'Internal server error' - ) - }) - }) - - return it('returns an error', function(done) { - return this.S3PersistorManager.getFileStream( - this.bucketName, - this.key, - {}, - (err, stream) => { - expect(err).to.be.instanceof(Error) - return done() - } - ) - }) + it('returns a stream', function() { + expect(stream).to.equal('s3Stream') }) - return describe('when the S3 request errors out before receiving HTTP headers', function() { - beforeEach(function() { - return this.s3Request.send.callsFake(() => { - return this.s3EventHandlers.error(new Error('connection failed')) - }) + it('passes the byte range on to S3', function() { + expect(S3Client.getObject).to.have.been.calledWith({ + Bucket: bucket, + Key: key, + Range: 'bytes=5-10' }) + }) + }) - return it('returns an error', function(done) { - return this.S3PersistorManager.getFileStream( - this.bucketName, - this.key, - {}, - (err, stream) => { - expect(err).to.be.instanceof(Error) - return done() - } - ) - }) + describe("when the file doesn't exist", function() { + let error, stream + + beforeEach(async function() { + S3ReadStream.on = sinon.stub() + S3ReadStream.on.withArgs('error').yields(S3NotFoundError) + try { + stream = await S3PersistorManager.promises.getFileStream(bucket, key) + console.log(stream) + } catch (err) { + error = err + } + }) + + it('does not return a stream', function() { + expect(stream).not.to.exist + }) + + it('throws a NotFoundError', function() { + expect(error).to.be.an.instanceOf(Errors.NotFoundError) + }) + + it('wraps the error from S3', function() { + expect(error.cause).to.equal(S3NotFoundError) + }) + + it('stores the bucket and key in the error', function() { + expect(error.info).to.deep.equal({ Bucket: bucket, Key: key }) + }) + }) + + describe('when S3 encounters an unkown error', function() { + let error, stream + + beforeEach(async function() { + S3ReadStream.on = sinon.stub() + S3ReadStream.on.withArgs('error').yields(genericError) + try { + stream = await S3PersistorManager.promises.getFileStream(bucket, key) + console.log(stream) + } catch (err) { + error = err + } + }) + + it('does not return a stream', function() { + expect(stream).not.to.exist + }) + + it('throws a ReadError', function() { + expect(error).to.be.an.instanceOf(Errors.ReadError) + }) + + it('wraps the error from S3', function() { + expect(error.cause).to.equal(genericError) + }) + + it('stores the bucket and key in the error', function() { + expect(error.info).to.deep.equal({ Bucket: bucket, Key: key }) }) }) }) describe('getFileSize', function() { - it('should obtain the file size from S3', function(done) { - const expectedFileSize = 123 - this.s3Client.headObject.yields( - new Error('s3Client.headObject got unexpected arguments') - ) - this.s3Client.headObject - .withArgs({ - Bucket: this.bucketName, - Key: this.key - }) - .yields(null, { ContentLength: expectedFileSize }) + describe('when called with valid parameters', function() { + let size - return this.S3PersistorManager.getFileSize( - this.bucketName, - this.key, - (err, fileSize) => { - if (err != null) { - return done(err) - } - expect(fileSize).to.equal(expectedFileSize) - return done() - } - ) - }) - ;[403, 404].forEach(statusCode => - it(`should throw NotFoundError when S3 responds with ${statusCode}`, function(done) { - const error = new Error() - error.statusCode = statusCode - this.s3Client.headObject.yields(error) - - return this.S3PersistorManager.getFileSize( - this.bucketName, - this.key, - (err, fileSize) => { - expect(err).to.be.an.instanceof(this.Errors.NotFoundError) - return done() - } - ) + beforeEach(async function() { + size = await S3PersistorManager.promises.getFileSize(bucket, key) }) - ) - return it('should rethrow any other error', function(done) { - const error = new Error() - this.s3Client.headObject.yields(error) - this.s3Client.headObject.yields(error) + it('should return the object size', function() { + expect(size).to.equal(objectSize) + }) - return this.S3PersistorManager.getFileSize( - this.bucketName, - this.key, - (err, fileSize) => { - expect(err).to.equal(error) - return done() - } - ) - }) - }) - - describe('sendFile', function() { - beforeEach(function() { - return this.knoxClient.putFile.returns({ on() {} }) + it('should pass the bucket and key to S3', function() { + expect(S3Client.headObject).to.have.been.calledWith({ + Bucket: bucket, + Key: key + }) + }) }) - it('should put file with knox', function(done) { - this.LocalFileWriter.deleteFile.callsArgWith(1) - this.knoxClient.putFile.callsArgWith(2, this.error) - return this.S3PersistorManager.sendFile( - this.bucketName, - this.key, - this.fsPath, - err => { - this.knoxClient.putFile - .calledWith(this.fsPath, this.key) - .should.equal(true) - err.should.equal(this.error) - return done() + describe('when the object is not found', function() { + let error + + beforeEach(async function() { + S3Client.headObject = sinon.stub().returns({ + promise: sinon.stub().rejects(S3NotFoundError) + }) + try { + await S3PersistorManager.promises.getFileSize(bucket, key) + } catch (err) { + error = err } - ) + }) + + it('should return a NotFoundError', function() { + expect(error).to.be.an.instanceOf(Errors.NotFoundError) + }) + + it('should wrap the error', function() { + expect(error.cause).to.equal(S3NotFoundError) + }) }) - return it('should delete the file and pass the error with it', function(done) { - this.LocalFileWriter.deleteFile.callsArgWith(1) - this.knoxClient.putFile.callsArgWith(2, this.error) - return this.S3PersistorManager.sendFile( - this.bucketName, - this.key, - this.fsPath, - err => { - this.knoxClient.putFile - .calledWith(this.fsPath, this.key) - .should.equal(true) - err.should.equal(this.error) - return done() + describe('when S3 returns an error', function() { + let error + + beforeEach(async function() { + S3Client.headObject = sinon.stub().returns({ + promise: sinon.stub().rejects(genericError) + }) + try { + await S3PersistorManager.promises.getFileSize(bucket, key) + } catch (err) { + error = err } - ) + }) + + it('should return a ReadError', function() { + expect(error).to.be.an.instanceOf(Errors.ReadError) + }) + + it('should wrap the error', function() { + expect(error.cause).to.equal(genericError) + }) }) }) describe('sendStream', function() { - beforeEach(function() { - this.fsPath = 'to/some/where' - this.origin = { on() {} } - return (this.S3PersistorManager.sendFile = sinon.stub().callsArgWith(3)) + describe('with valid parameters', function() { + beforeEach(async function() { + return S3PersistorManager.promises.sendStream(bucket, key, ReadStream) + }) + + it('should upload the stream', function() { + expect(S3Client.upload).to.have.been.calledWith({ + Bucket: bucket, + Key: key, + Body: 'readStream' + }) + }) + + it('should meter the stream', function() { + expect(ReadStream.pipe).to.have.been.calledWith(MeteredStream) + }) + + it('should record an egress metric', function() { + expect(Metrics.count).to.have.been.calledWith('s3.egress', objectSize) + }) }) - it('should send stream to LocalFileWriter', function(done) { - this.LocalFileWriter.deleteFile.callsArgWith(1) - this.LocalFileWriter.writeStream.callsArgWith(2, null, this.fsPath) - return this.S3PersistorManager.sendStream( - this.bucketName, - this.key, - this.origin, - () => { - this.LocalFileWriter.writeStream - .calledWith(this.origin) - .should.equal(true) - return done() + describe('when the upload fails', function() { + let error + beforeEach(async function() { + S3Client.upload = sinon.stub().returns({ + promise: sinon.stub().rejects(genericError) + }) + try { + await S3PersistorManager.promises.sendStream(bucket, key, ReadStream) + } catch (err) { + error = err } - ) + }) + + it('throws a WriteError', function() { + expect(error).to.be.an.instanceOf(Errors.WriteError) + }) + }) + }) + + describe('sendFile', function() { + describe('with valid parameters', function() { + beforeEach(async function() { + return S3PersistorManager.promises.sendFile(bucket, key, filename) + }) + + it('should create a read stream for the file', function() { + expect(Fs.createReadStream).to.have.been.calledWith(filename) + }) + + it('should upload the stream', function() { + expect(S3Client.upload).to.have.been.calledWith({ + Bucket: bucket, + Key: key, + Body: 'readStream' + }) + }) }) - it('should return the error from LocalFileWriter', function(done) { - this.LocalFileWriter.deleteFile.callsArgWith(1) - this.LocalFileWriter.writeStream.callsArgWith(2, this.error) - return this.S3PersistorManager.sendStream( - this.bucketName, - this.key, - this.origin, - err => { - err.should.equal(this.error) - return done() + describe('when the file does not exist', function() { + let error + + beforeEach(async function() { + Fs.createReadStream = sinon.stub().throws(FileNotFoundError) + try { + await S3PersistorManager.promises.sendFile(bucket, key, filename) + } catch (err) { + error = err } - ) + }) + + it('returns a NotFoundError', function() { + expect(error).to.be.an.instanceOf(Errors.NotFoundError) + }) + + it('wraps the error', function() { + expect(error.cause).to.equal(FileNotFoundError) + }) }) - return it('should send the file to the filestore', function(done) { - this.LocalFileWriter.deleteFile.callsArgWith(1) - this.LocalFileWriter.writeStream.callsArgWith(2) - return this.S3PersistorManager.sendStream( - this.bucketName, - this.key, - this.origin, - err => { - this.S3PersistorManager.sendFile.called.should.equal(true) - return done() + describe('when reading the file throws an error', function() { + let error + + beforeEach(async function() { + Fs.createReadStream = sinon.stub().throws(genericError) + try { + await S3PersistorManager.promises.sendFile(bucket, key, filename) + } catch (err) { + error = err } - ) + }) + + it('returns a ReadError', function() { + expect(error).to.be.an.instanceOf(Errors.ReadError) + }) + + it('wraps the error', function() { + expect(error.cause).to.equal(genericError) + }) }) }) describe('copyFile', function() { - beforeEach(function() { - this.sourceKey = 'my/key' - return (this.destKey = 'my/dest/key') + describe('with valid parameters', function() { + beforeEach(async function() { + return S3PersistorManager.promises.copyFile(bucket, key, destKey) + }) + + it('should copy the object', function() { + expect(S3Client.copyObject).to.have.been.calledWith({ + Bucket: bucket, + Key: destKey, + CopySource: `${bucket}/${key}` + }) + }) }) - it('should use AWS SDK to copy file', function(done) { - this.s3Client.copyObject.callsArgWith(1, this.error) - return this.S3PersistorManager.copyFile( - this.bucketName, - this.sourceKey, - this.destKey, - err => { - err.should.equal(this.error) - this.s3Client.copyObject - .calledWith({ - Bucket: this.bucketName, - Key: this.destKey, - CopySource: this.bucketName + '/' + this.key - }) - .should.equal(true) - return done() - } - ) - }) + describe('when the file does not exist', function() { + let error - return it('should return a NotFoundError object if the original file does not exist', function(done) { - const NoSuchKeyError = { code: 'NoSuchKey' } - this.s3Client.copyObject.callsArgWith(1, NoSuchKeyError) - return this.S3PersistorManager.copyFile( - this.bucketName, - this.sourceKey, - this.destKey, - err => { - expect(err instanceof this.Errors.NotFoundError).to.equal(true) - return done() + beforeEach(async function() { + S3Client.copyObject = sinon.stub().returns({ + promise: sinon.stub().rejects(S3NotFoundError) + }) + try { + await S3PersistorManager.promises.copyFile(bucket, key, destKey) + } catch (err) { + error = err } - ) + }) + + it('should throw a NotFoundError', function() { + expect(error).to.be.an.instanceOf(Errors.NotFoundError) + }) }) }) - describe('deleteDirectory', () => - it('should list the contents passing them onto multi delete', function(done) { - const data = { Contents: [{ Key: '1234' }, { Key: '456' }] } - this.knoxClient.list.callsArgWith(1, null, data) - this.knoxClient.deleteMultiple.callsArgWith(1) - return this.S3PersistorManager.deleteDirectory( - this.bucketName, - this.key, - err => { - this.knoxClient.deleteMultiple - .calledWith(['1234', '456']) - .should.equal(true) - return done() - } - ) - })) - describe('deleteFile', function() { - it('should use correct options', function(done) { - this.request.callsArgWith(1) + describe('with valid parameters', function() { + beforeEach(async function() { + return S3PersistorManager.promises.deleteFile(bucket, key) + }) - return this.S3PersistorManager.deleteFile( - this.bucketName, - this.key, - err => { - const opts = this.request.args[0][0] - assert.deepEqual(opts.aws, { - key: this.settings.filestore.s3.key, - secret: this.settings.filestore.s3.secret, - bucket: this.bucketName - }) - opts.method.should.equal('delete') - opts.timeout.should.equal(30 * 1000) - opts.uri.should.equal( - `https://${this.bucketName}.s3.amazonaws.com/${this.key}` - ) - return done() - } - ) + it('should delete the object', function() { + expect(S3Client.deleteObject).to.have.been.calledWith({ + Bucket: bucket, + Key: key + }) + }) }) - return it('should return the error', function(done) { - this.request.callsArgWith(1, this.error) + describe('when the file does not exist', function() { + let error - return this.S3PersistorManager.deleteFile( - this.bucketName, - this.key, - err => { - err.should.equal(this.error) - return done() + beforeEach(async function() { + S3Client.deleteObject = sinon.stub().returns({ + promise: sinon.stub().rejects(S3NotFoundError) + }) + try { + await S3PersistorManager.promises.deleteFile(bucket, key) + } catch (err) { + error = err } - ) + }) + + it('should throw a NotFoundError', function() { + expect(error).to.be.an.instanceOf(Errors.NotFoundError) + }) + }) + }) + + describe('deleteDirectory', function() { + describe('with valid parameters', function() { + beforeEach(async function() { + return S3PersistorManager.promises.deleteDirectory(bucket, key) + }) + + it('should list the objects in the directory', function() { + expect(S3Client.listObjects).to.have.been.calledWith({ + Bucket: bucket, + Prefix: key + }) + }) + + it('should delete the objects using their keys', function() { + expect(S3Client.deleteObjects).to.have.been.calledWith({ + Bucket: bucket, + Delete: { + Objects: [{ Key: 'llama' }, { Key: 'hippo' }], + Quiet: true + } + }) + }) + }) + + describe('when there are no files', function() { + beforeEach(async function() { + S3Client.listObjects = sinon + .stub() + .returns({ promise: sinon.stub().resolves({ Contents: [] }) }) + return S3PersistorManager.promises.deleteDirectory(bucket, key) + }) + + it('should list the objects in the directory', function() { + expect(S3Client.listObjects).to.have.been.calledWith({ + Bucket: bucket, + Prefix: key + }) + }) + + it('should not try to delete any objects', function() { + expect(S3Client.deleteObjects).not.to.have.been.called + }) + }) + + describe('when there is an error listing the objects', function() { + let error + + beforeEach(async function() { + S3Client.listObjects = sinon + .stub() + .returns({ promise: sinon.stub().rejects(genericError) }) + try { + await S3PersistorManager.promises.deleteDirectory(bucket, key) + } catch (err) { + error = err + } + }) + + it('should generate a ReadError', function() { + expect(error).to.be.an.instanceOf(Errors.ReadError) + }) + + it('should wrap the error', function() { + expect(error.cause).to.equal(genericError) + }) + + it('should not try to delete any objects', function() { + expect(S3Client.deleteObjects).not.to.have.been.called + }) + }) + + describe('when there is an error deleting the objects', function() { + let error + + beforeEach(async function() { + S3Client.deleteObjects = sinon + .stub() + .returns({ promise: sinon.stub().rejects(genericError) }) + try { + await S3PersistorManager.promises.deleteDirectory(bucket, key) + } catch (err) { + error = err + } + }) + + it('should generate a WriteError', function() { + expect(error).to.be.an.instanceOf(Errors.WriteError) + }) + + it('should wrap the error', function() { + expect(error.cause).to.equal(genericError) + }) + }) + }) + + describe('directorySize', function() { + describe('with valid parameters', function() { + let size + + beforeEach(async function() { + size = await S3PersistorManager.promises.getDirectorySize(bucket, key) + }) + + it('should list the objects in the directory', function() { + expect(S3Client.listObjects).to.have.been.calledWith({ + Bucket: bucket, + Prefix: key + }) + }) + + it('should return the directory size', function() { + expect(size).to.equal(filesSize) + }) + }) + + describe('when there are no files', function() { + let size + + beforeEach(async function() { + S3Client.listObjects = sinon + .stub() + .returns({ promise: sinon.stub().resolves({ Contents: [] }) }) + size = await S3PersistorManager.promises.getDirectorySize(bucket, key) + }) + + it('should list the objects in the directory', function() { + expect(S3Client.listObjects).to.have.been.calledWith({ + Bucket: bucket, + Prefix: key + }) + }) + + it('should return zero', function() { + expect(size).to.equal(0) + }) + }) + + describe('when there is an error listing the objects', function() { + let error + + beforeEach(async function() { + S3Client.listObjects = sinon + .stub() + .returns({ promise: sinon.stub().rejects(genericError) }) + try { + await S3PersistorManager.promises.getDirectorySize(bucket, key) + } catch (err) { + error = err + } + }) + + it('should generate a ReadError', function() { + expect(error).to.be.an.instanceOf(Errors.ReadError) + }) + + it('should wrap the error', function() { + expect(error.cause).to.equal(genericError) + }) }) }) describe('checkIfFileExists', function() { - it('should use correct options', function(done) { - this.request.callsArgWith(1, null, { statusCode: 200 }) + describe('when the file exists', function() { + let exists - return this.S3PersistorManager.checkIfFileExists( - this.bucketName, - this.key, - err => { - const opts = this.request.args[0][0] - assert.deepEqual(opts.aws, { - key: this.settings.filestore.s3.key, - secret: this.settings.filestore.s3.secret, - bucket: this.bucketName - }) - opts.method.should.equal('head') - opts.timeout.should.equal(30 * 1000) - opts.uri.should.equal( - `https://${this.bucketName}.s3.amazonaws.com/${this.key}` - ) - return done() - } - ) + beforeEach(async function() { + exists = await S3PersistorManager.promises.checkIfFileExists( + bucket, + key + ) + }) + + it('should get the object header', function() { + expect(S3Client.headObject).to.have.been.calledWith({ + Bucket: bucket, + Key: key + }) + }) + + it('should return that the file exists', function() { + expect(exists).to.equal(true) + }) }) - it('should return true for a 200', function(done) { - this.request.callsArgWith(1, null, { statusCode: 200 }) + describe('when the file does not exist', function() { + let exists - return this.S3PersistorManager.checkIfFileExists( - this.bucketName, - this.key, - (err, exists) => { - exists.should.equal(true) - return done() - } - ) + beforeEach(async function() { + S3Client.headObject = sinon + .stub() + .returns({ promise: sinon.stub().rejects(S3NotFoundError) }) + exists = await S3PersistorManager.promises.checkIfFileExists( + bucket, + key + ) + }) + + it('should get the object header', function() { + expect(S3Client.headObject).to.have.been.calledWith({ + Bucket: bucket, + Key: key + }) + }) + + it('should return that the file does not exist', function() { + expect(exists).to.equal(false) + }) }) - it('should return false for a non 200', function(done) { - this.request.callsArgWith(1, null, { statusCode: 404 }) + describe('when there is an error', function() { + let error - return this.S3PersistorManager.checkIfFileExists( - this.bucketName, - this.key, - (err, exists) => { - exists.should.equal(false) - return done() + beforeEach(async function() { + S3Client.headObject = sinon + .stub() + .returns({ promise: sinon.stub().rejects(genericError) }) + try { + await S3PersistorManager.promises.checkIfFileExists(bucket, key) + } catch (err) { + error = err } - ) - }) + }) - return it('should return the error', function(done) { - this.request.callsArgWith(1, this.error, {}) + it('should generate a ReadError', function() { + expect(error).to.be.an.instanceOf(Errors.ReadError) + }) - return this.S3PersistorManager.checkIfFileExists( - this.bucketName, - this.key, - err => { - err.should.equal(this.error) - return done() - } - ) + it('should wrap the upstream ReadError', function() { + expect(error.cause).to.be.an.instanceOf(Errors.ReadError) + }) + + it('should eventually wrap the error', function() { + expect(error.cause.cause).to.equal(genericError) + }) }) }) - - return describe('directorySize', () => - it('should sum directory files size', function(done) { - const data = { Contents: [{ Size: 1024 }, { Size: 2048 }] } - this.knoxClient.list.callsArgWith(1, null, data) - return this.S3PersistorManager.directorySize( - this.bucketName, - this.key, - (err, totalSize) => { - totalSize.should.equal(3072) - return done() - } - ) - })) }) From a6d9d34fe3b2758b7791878cce5df75ea215fdf2 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Sat, 4 Jan 2020 17:00:04 +0000 Subject: [PATCH 66/73] Configure PersistorManager to use the new S3 manager for 'aws-sdk' --- services/filestore/app/js/PersistorManager.js | 2 -- services/filestore/test/unit/js/PersistorManagerTests.js | 8 ++++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/services/filestore/app/js/PersistorManager.js b/services/filestore/app/js/PersistorManager.js index 8124d66101..cca0cf0f36 100644 --- a/services/filestore/app/js/PersistorManager.js +++ b/services/filestore/app/js/PersistorManager.js @@ -13,8 +13,6 @@ if (!settings.filestore.backend) { switch (settings.filestore.backend) { case 'aws-sdk': - module.exports = require('./AWSSDKPersistorManager') - break case 's3': module.exports = require('./S3PersistorManager') break diff --git a/services/filestore/test/unit/js/PersistorManagerTests.js b/services/filestore/test/unit/js/PersistorManagerTests.js index d8fd887265..0ecbb22078 100644 --- a/services/filestore/test/unit/js/PersistorManagerTests.js +++ b/services/filestore/test/unit/js/PersistorManagerTests.js @@ -43,6 +43,14 @@ describe('PersistorManager', function() { expect(PersistorManager.wrappedMethod()).to.equal('S3PersistorManager') }) + it("should implement the S3 wrapped method when 'aws-sdk' is configured", function() { + settings.filestore.backend = 'aws-sdk' + PersistorManager = SandboxedModule.require(modulePath, { requires }) + + expect(PersistorManager).to.respondTo('wrappedMethod') + expect(PersistorManager.wrappedMethod()).to.equal('S3PersistorManager') + }) + it('should implement the FS wrapped method when FS is configured', function() { settings.filestore.backend = 'fs' PersistorManager = SandboxedModule.require(modulePath, { requires }) From b642b3cf3a418daa25300c1e796e5004537346c3 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 6 Jan 2020 15:35:40 +0000 Subject: [PATCH 67/73] Rename incorrectly-named 'getDirectorySize' method --- services/filestore/app/js/S3PersistorManager.js | 6 +++--- .../filestore/test/unit/js/S3PersistorManagerTests.js | 8 +++----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/services/filestore/app/js/S3PersistorManager.js b/services/filestore/app/js/S3PersistorManager.js index 5f8bb34c5d..d0729b80b9 100644 --- a/services/filestore/app/js/S3PersistorManager.js +++ b/services/filestore/app/js/S3PersistorManager.js @@ -23,7 +23,7 @@ module.exports = { deleteFile: callbackify(deleteFile), copyFile: callbackify(copyFile), checkIfFileExists: callbackify(checkIfFileExists), - getDirectorySize: callbackify(getDirectorySize), + directorySize: callbackify(directorySize), promises: { sendFile, sendStream, @@ -33,7 +33,7 @@ module.exports = { deleteFile, copyFile, checkIfFileExists, - getDirectorySize + directorySize } } @@ -209,7 +209,7 @@ async function checkIfFileExists(bucketName, key) { } } -async function getDirectorySize(bucketName, key) { +async function directorySize(bucketName, key) { try { const response = await _client .listObjects({ Bucket: bucketName, Prefix: key }) diff --git a/services/filestore/test/unit/js/S3PersistorManagerTests.js b/services/filestore/test/unit/js/S3PersistorManagerTests.js index 3caf2db73d..2faa8b52c7 100644 --- a/services/filestore/test/unit/js/S3PersistorManagerTests.js +++ b/services/filestore/test/unit/js/S3PersistorManagerTests.js @@ -186,7 +186,6 @@ describe('S3PersistorManagerTests', function() { S3ReadStream.on.withArgs('error').yields(S3NotFoundError) try { stream = await S3PersistorManager.promises.getFileStream(bucket, key) - console.log(stream) } catch (err) { error = err } @@ -217,7 +216,6 @@ describe('S3PersistorManagerTests', function() { S3ReadStream.on.withArgs('error').yields(genericError) try { stream = await S3PersistorManager.promises.getFileStream(bucket, key) - console.log(stream) } catch (err) { error = err } @@ -581,7 +579,7 @@ describe('S3PersistorManagerTests', function() { let size beforeEach(async function() { - size = await S3PersistorManager.promises.getDirectorySize(bucket, key) + size = await S3PersistorManager.promises.directorySize(bucket, key) }) it('should list the objects in the directory', function() { @@ -603,7 +601,7 @@ describe('S3PersistorManagerTests', function() { S3Client.listObjects = sinon .stub() .returns({ promise: sinon.stub().resolves({ Contents: [] }) }) - size = await S3PersistorManager.promises.getDirectorySize(bucket, key) + size = await S3PersistorManager.promises.directorySize(bucket, key) }) it('should list the objects in the directory', function() { @@ -626,7 +624,7 @@ describe('S3PersistorManagerTests', function() { .stub() .returns({ promise: sinon.stub().rejects(genericError) }) try { - await S3PersistorManager.promises.getDirectorySize(bucket, key) + await S3PersistorManager.promises.directorySize(bucket, key) } catch (err) { error = err } From ffd002dc76341d6ff9494a83796093410b37c20e Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Mon, 6 Jan 2020 15:47:46 +0000 Subject: [PATCH 68/73] Add explanatory comment --- services/filestore/app/js/FileController.js | 1 + 1 file changed, 1 insertion(+) diff --git a/services/filestore/app/js/FileController.js b/services/filestore/app/js/FileController.js index 726438a557..eef441436c 100644 --- a/services/filestore/app/js/FileController.js +++ b/services/filestore/app/js/FileController.js @@ -62,6 +62,7 @@ function getFile(req, res, next) { logger.log({ key, bucket, format, style }, 'sending file to response') + // pass 'next' as a callback to 'pipeline' to receive any errors pipeline(fileStream, res, next) }) } From 2bb2caf7b3fa8aac1a6148ddbb23f8b4c12d0efc Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Tue, 7 Jan 2020 09:46:53 +0000 Subject: [PATCH 69/73] Clean up settings tests --- .../filestore/test/unit/js/SettingsTests.js | 32 ++++++------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/services/filestore/test/unit/js/SettingsTests.js b/services/filestore/test/unit/js/SettingsTests.js index 472c6d1179..4563449fde 100644 --- a/services/filestore/test/unit/js/SettingsTests.js +++ b/services/filestore/test/unit/js/SettingsTests.js @@ -1,33 +1,19 @@ -/* eslint-disable - camelcase, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const { assert } = require('chai') -const sinon = require('sinon') const chai = require('chai') -const should = chai.should() const { expect } = chai -const modulePath = '../../../app/js/BucketController.js' -describe('Settings', () => - describe('s3', () => - it('should use JSONified env var if present', function(done) { - const s3_settings = { +describe('Settings', function() { + describe('s3', function() { + it('should use JSONified env var if present', function() { + const s3Settings = { bucket1: { auth_key: 'bucket1_key', auth_secret: 'bucket1_secret' } } - process.env.S3_BUCKET_CREDENTIALS = JSON.stringify(s3_settings) + process.env.S3_BUCKET_CREDENTIALS = JSON.stringify(s3Settings) const settings = require('settings-sharelatex') - expect(settings.filestore.s3BucketCreds).to.deep.equal(s3_settings) - return done() - }))) + expect(settings.filestore.s3BucketCreds).to.deep.equal(s3Settings) + }) + }) +}) From 6cc5d94f13bb8c5328579e8226171f2c85addc38 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Tue, 7 Jan 2020 10:24:46 +0000 Subject: [PATCH 70/73] Re-add bucket-specific credentials support for S3 --- services/filestore/app/js/Errors.js | 4 +- .../filestore/app/js/S3PersistorManager.js | 77 +++++++++--- .../test/unit/js/S3PersistorManagerTests.js | 117 +++++++++++++++--- 3 files changed, 166 insertions(+), 32 deletions(-) diff --git a/services/filestore/app/js/Errors.js b/services/filestore/app/js/Errors.js index 65af6dc056..06091a13ba 100644 --- a/services/filestore/app/js/Errors.js +++ b/services/filestore/app/js/Errors.js @@ -23,6 +23,7 @@ class ReadError extends BackwardCompatibleError {} class HealthCheckError extends BackwardCompatibleError {} class ConversionsDisabledError extends BackwardCompatibleError {} class ConversionError extends BackwardCompatibleError {} +class SettingsError extends BackwardCompatibleError {} class FailedCommandError extends OError { constructor(command, code, stdout, stderr) { @@ -46,5 +47,6 @@ module.exports = { WriteError, ReadError, ConversionError, - HealthCheckError + HealthCheckError, + SettingsError } diff --git a/services/filestore/app/js/S3PersistorManager.js b/services/filestore/app/js/S3PersistorManager.js index d0729b80b9..36c49d35bd 100644 --- a/services/filestore/app/js/S3PersistorManager.js +++ b/services/filestore/app/js/S3PersistorManager.js @@ -12,7 +12,12 @@ const fs = require('fs') const S3 = require('aws-sdk/clients/s3') const { URL } = require('url') const { callbackify } = require('util') -const { WriteError, ReadError, NotFoundError } = require('./Errors') +const { + WriteError, + ReadError, + NotFoundError, + SettingsError +} = require('./Errors') module.exports = { sendFile: callbackify(sendFile), @@ -37,8 +42,6 @@ module.exports = { } } -const _client = new S3(_defaultOptions()) - async function sendFile(bucketName, key, fsPath) { let readStream try { @@ -61,7 +64,7 @@ async function sendStream(bucketName, key, readStream) { metrics.count('s3.egress', meteredStream.bytes) }) - const response = await _client + const response = await _client(bucketName) .upload({ Bucket: bucketName, Key: key, @@ -92,7 +95,9 @@ async function getFileStream(bucketName, key, opts) { } return new Promise((resolve, reject) => { - const stream = _client.getObject(params).createReadStream() + const stream = _client(bucketName) + .getObject(params) + .createReadStream() const meteredStream = meter() meteredStream.on('finish', () => { @@ -115,7 +120,7 @@ async function deleteDirectory(bucketName, key) { let response try { - response = await _client + response = await _client(bucketName) .listObjects({ Bucket: bucketName, Prefix: key }) .promise() } catch (err) { @@ -130,7 +135,7 @@ async function deleteDirectory(bucketName, key) { const objects = response.Contents.map(item => ({ Key: item.Key })) if (objects.length) { try { - await _client + await _client(bucketName) .deleteObjects({ Bucket: bucketName, Delete: { @@ -152,7 +157,7 @@ async function deleteDirectory(bucketName, key) { async function getFileSize(bucketName, key) { try { - const response = await _client + const response = await _client(bucketName) .headObject({ Bucket: bucketName, Key: key }) .promise() return response.ContentLength @@ -168,7 +173,9 @@ async function getFileSize(bucketName, key) { async function deleteFile(bucketName, key) { try { - await _client.deleteObject({ Bucket: bucketName, Key: key }).promise() + await _client(bucketName) + .deleteObject({ Bucket: bucketName, Key: key }) + .promise() } catch (err) { throw _wrapError( err, @@ -186,7 +193,9 @@ async function copyFile(bucketName, sourceKey, destKey) { CopySource: `${bucketName}/${sourceKey}` } try { - await _client.copyObject(params).promise() + await _client(bucketName) + .copyObject(params) + .promise() } catch (err) { throw _wrapError(err, 'failed to copy file in S3', params, WriteError) } @@ -211,7 +220,7 @@ async function checkIfFileExists(bucketName, key) { async function directorySize(bucketName, key) { try { - const response = await _client + const response = await _client(bucketName) .listObjects({ Bucket: bucketName, Prefix: key }) .promise() @@ -240,9 +249,49 @@ function _wrapError(error, message, params, ErrorType) { } } -function _defaultOptions() { - const options = { - credentials: { +const _clients = {} + +function _client(bucket) { + if (_clients[bucket]) { + return _clients[bucket] + } + + if ( + settings.filestore.s3.s3BucketCreds && + settings.filestore.s3.s3BucketCreds[bucket] + ) { + _clients[bucket] = new S3( + _clientOptions(settings.filestore.s3.s3BucketCreds[bucket]) + ) + return _clients[bucket] + } + + // no specific credentials for the bucket + if (_clients.default) { + return _clients.default + } + + if (settings.filestore.s3.key) { + _clients.default = new S3(_clientOptions()) + return _clients.default + } + + throw new SettingsError({ + message: 'no bucket-specific or default credentials provided', + info: { bucket } + }) +} + +function _clientOptions(bucketCredentials) { + const options = {} + + if (bucketCredentials) { + options.credentials = { + accessKeyId: bucketCredentials.auth_key, + secretAccessKey: bucketCredentials.auth_secret + } + } else { + options.credentials = { accessKeyId: settings.filestore.s3.key, secretAccessKey: settings.filestore.s3.secret } diff --git a/services/filestore/test/unit/js/S3PersistorManagerTests.js b/services/filestore/test/unit/js/S3PersistorManagerTests.js index 2faa8b52c7..bdac7c8232 100644 --- a/services/filestore/test/unit/js/S3PersistorManagerTests.js +++ b/services/filestore/test/unit/js/S3PersistorManagerTests.js @@ -7,16 +7,12 @@ const SandboxedModule = require('sandboxed-module') const Errors = require('../../../app/js/Errors') describe('S3PersistorManagerTests', function() { - const settings = { - filestore: { - backend: 's3', - s3: { - secret: 'secret', - key: 'this_key' - }, - stores: { - user_files: 'sl_user_files' - } + const defaultS3Key = 'frog' + const defaultS3Secret = 'prince' + const defaultS3Credentials = { + credentials: { + accessKeyId: defaultS3Key, + secretAccessKey: defaultS3Secret } } const filename = '/wombat/potato.tex' @@ -42,9 +38,23 @@ describe('S3PersistorManagerTests', function() { S3ReadStream, S3NotFoundError, FileNotFoundError, - EmptyPromise + EmptyPromise, + settings beforeEach(function() { + settings = { + filestore: { + backend: 's3', + s3: { + secret: defaultS3Secret, + key: defaultS3Key + }, + stores: { + user_files: 'sl_user_files' + } + } + } + EmptyPromise = { promise: sinon.stub().resolves() } @@ -131,12 +141,7 @@ describe('S3PersistorManagerTests', function() { }) it('sets the AWS client up with credentials from settings', function() { - expect(S3).to.have.been.calledWith({ - credentials: { - accessKeyId: settings.filestore.s3.key, - secretAccessKey: settings.filestore.s3.secret - } - }) + expect(S3).to.have.been.calledWith(defaultS3Credentials) }) it('fetches the right key from the right bucket', function() { @@ -178,6 +183,84 @@ describe('S3PersistorManagerTests', function() { }) }) + describe('when there are alternative credentials', function() { + let stream + const alternativeSecret = 'giraffe' + const alternativeKey = 'hippo' + const alternativeS3Credentials = { + credentials: { + accessKeyId: alternativeKey, + secretAccessKey: alternativeSecret + } + } + + beforeEach(async function() { + settings.filestore.s3.s3BucketCreds = {} + settings.filestore.s3.s3BucketCreds[bucket] = { + auth_key: alternativeKey, + auth_secret: alternativeSecret + } + + stream = await S3PersistorManager.promises.getFileStream(bucket, key) + }) + + it('returns a stream', function() { + expect(stream).to.equal('s3Stream') + }) + + it('sets the AWS client up with the alternative credentials', function() { + expect(S3).to.have.been.calledWith(alternativeS3Credentials) + }) + + it('fetches the right key from the right bucket', function() { + expect(S3Client.getObject).to.have.been.calledWith({ + Bucket: bucket, + Key: key + }) + }) + + it('caches the credentials', async function() { + stream = await S3PersistorManager.promises.getFileStream(bucket, key) + + expect(S3).to.have.been.calledOnceWith(alternativeS3Credentials) + }) + + it('uses the default credentials for an unknown bucket', async function() { + stream = await S3PersistorManager.promises.getFileStream( + 'anotherBucket', + key + ) + + expect(S3).to.have.been.calledTwice + expect(S3.firstCall).to.have.been.calledWith(alternativeS3Credentials) + expect(S3.secondCall).to.have.been.calledWith(defaultS3Credentials) + }) + + it('caches the default credentials', async function() { + stream = await S3PersistorManager.promises.getFileStream( + 'anotherBucket', + key + ) + stream = await S3PersistorManager.promises.getFileStream( + 'anotherBucket', + key + ) + + expect(S3).to.have.been.calledTwice + expect(S3.firstCall).to.have.been.calledWith(alternativeS3Credentials) + expect(S3.secondCall).to.have.been.calledWith(defaultS3Credentials) + }) + + it('throws an error if there are no credentials for the bucket', async function() { + delete settings.filestore.s3.key + delete settings.filestore.s3.secret + + await expect( + S3PersistorManager.promises.getFileStream('anotherBucket', key) + ).to.eventually.be.rejected.and.be.an.instanceOf(Errors.SettingsError) + }) + }) + describe("when the file doesn't exist", function() { let error, stream From 80d41cf51bf7d3ed7f7202793c41454fc31b6b82 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Tue, 7 Jan 2020 15:05:51 +0000 Subject: [PATCH 71/73] Move bucket-specific file endpoint into FileController --- services/filestore/app.js | 7 +- services/filestore/app/js/BucketController.js | 48 --------- services/filestore/app/js/KeyBuilder.js | 7 ++ .../filestore/app/js/S3PersistorManager.js | 5 + .../test/acceptance/js/FilestoreTests.js | 46 +++++++- .../test/unit/js/BucketControllerTests.js | 100 ------------------ 6 files changed, 62 insertions(+), 151 deletions(-) delete mode 100644 services/filestore/app/js/BucketController.js delete mode 100644 services/filestore/test/unit/js/BucketControllerTests.js diff --git a/services/filestore/app.js b/services/filestore/app.js index 232c5b24bc..9256cb0029 100644 --- a/services/filestore/app.js +++ b/services/filestore/app.js @@ -9,7 +9,6 @@ const express = require('express') const bodyParser = require('body-parser') const fileController = require('./app/js/FileController') -const bucketController = require('./app/js/BucketController') const keyBuilder = require('./app/js/KeyBuilder') const healthCheckController = require('./app/js/HealthCheckController') @@ -114,7 +113,11 @@ app.get( fileController.directorySize ) -app.get('/bucket/:bucket/key/*', bucketController.getFile) +app.get( + '/bucket/:bucket/key/*', + keyBuilder.bucketFileKeyMiddleware, + fileController.getFile +) app.get('/heapdump', (req, res, next) => require('heapdump').writeSnapshot( diff --git a/services/filestore/app/js/BucketController.js b/services/filestore/app/js/BucketController.js deleted file mode 100644 index 46f69679aa..0000000000 --- a/services/filestore/app/js/BucketController.js +++ /dev/null @@ -1,48 +0,0 @@ -/* eslint-disable - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -let BucketController -const settings = require('settings-sharelatex') -const logger = require('logger-sharelatex') -const FileHandler = require('./FileHandler') -const metrics = require('metrics-sharelatex') -const Errors = require('./Errors') - -module.exports = BucketController = { - getFile(req, res) { - const { bucket } = req.params - const key = req.params[0] - const credentials = - settings.filestore.s3BucketCreds != null - ? settings.filestore.s3BucketCreds[bucket] - : undefined - const options = { - key, - bucket, - credentials - } - metrics.inc(`${bucket}.getFile`) - logger.log({ key, bucket }, 'receiving request to get file from bucket') - return FileHandler.getFile(bucket, key, options, function(err, fileStream) { - if (err != null) { - logger.err({ err, key, bucket }, 'problem getting file from bucket') - if (err instanceof Errors.NotFoundError) { - return res.send(404) - } else { - return res.send(500) - } - } else { - logger.log({ key, bucket }, 'sending bucket file to response') - return fileStream.pipe(res) - } - }) - } -} diff --git a/services/filestore/app/js/KeyBuilder.js b/services/filestore/app/js/KeyBuilder.js index 8de7c0be2a..66cf563014 100644 --- a/services/filestore/app/js/KeyBuilder.js +++ b/services/filestore/app/js/KeyBuilder.js @@ -6,6 +6,7 @@ module.exports = { userFileKeyMiddleware, publicFileKeyMiddleware, publicProjectKeyMiddleware, + bucketFileKeyMiddleware, templateFileKeyMiddleware } @@ -48,6 +49,12 @@ function publicFileKeyMiddleware(req, res, next) { next() } +function bucketFileKeyMiddleware(req, res, next) { + req.bucket = req.params.bucket + req.key = req.params[0] + next() +} + function templateFileKeyMiddleware(req, res, next) { const { template_id: templateId, diff --git a/services/filestore/app/js/S3PersistorManager.js b/services/filestore/app/js/S3PersistorManager.js index 36c49d35bd..00ed46379b 100644 --- a/services/filestore/app/js/S3PersistorManager.js +++ b/services/filestore/app/js/S3PersistorManager.js @@ -303,5 +303,10 @@ function _clientOptions(bucketCredentials) { options.sslEnabled = endpoint.protocol === 'https' } + // path-style access is only used for acceptance tests + if (settings.filestore.s3.pathStyle) { + options.s3ForcePathStyle = true + } + return options } diff --git a/services/filestore/test/acceptance/js/FilestoreTests.js b/services/filestore/test/acceptance/js/FilestoreTests.js index 9260b1bd62..d7dfbce57c 100644 --- a/services/filestore/test/acceptance/js/FilestoreTests.js +++ b/services/filestore/test/acceptance/js/FilestoreTests.js @@ -7,6 +7,7 @@ const FilestoreApp = require('./FilestoreApp') const rp = require('request-promise-native').defaults({ resolveWithFullResponse: true }) +const S3 = require('aws-sdk/clients/s3') const Stream = require('stream') const request = require('request') const { promisify } = require('util') @@ -43,7 +44,8 @@ if (process.env.AWS_ACCESS_KEY_ID) { s3: { key: process.env.AWS_ACCESS_KEY_ID, secret: process.env.AWS_SECRET_ACCESS_KEY, - endpoint: process.env.AWS_S3_ENDPOINT + endpoint: process.env.AWS_S3_ENDPOINT, + pathStyle: true }, stores: { user_files: process.env.AWS_S3_USER_FILES_BUCKET_NAME, @@ -288,6 +290,48 @@ describe('Filestore', function() { }) }) + if (backend === 'S3Persistor') { + describe('with a file in a specific bucket', function() { + let constantFileContents, fileId, fileUrl, bucketName + + beforeEach(async function() { + constantFileContents = `This is a file in a different S3 bucket ${Math.random()}` + fileId = Math.random().toString() + bucketName = Math.random().toString() + fileUrl = `${filestoreUrl}/bucket/${bucketName}/key/${fileId}` + + const s3ClientSettings = { + credentials: { + accessKeyId: 'fake', + secretAccessKey: 'fake' + }, + endpoint: process.env.AWS_S3_ENDPOINT, + sslEnabled: false, + s3ForcePathStyle: true + } + + const s3 = new S3(s3ClientSettings) + await s3 + .createBucket({ + Bucket: bucketName + }) + .promise() + await s3 + .upload({ + Bucket: bucketName, + Key: fileId, + Body: constantFileContents + }) + .promise() + }) + + it('should get the file from the specified bucket', async function() { + const response = await rp.get(fileUrl) + expect(response.body).to.equal(constantFileContents) + }) + }) + } + describe('with a pdf file', function() { let fileId, fileUrl, localFileSize const localFileReadPath = Path.resolve( diff --git a/services/filestore/test/unit/js/BucketControllerTests.js b/services/filestore/test/unit/js/BucketControllerTests.js deleted file mode 100644 index ef74b3f6c0..0000000000 --- a/services/filestore/test/unit/js/BucketControllerTests.js +++ /dev/null @@ -1,100 +0,0 @@ -/* eslint-disable - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -const { assert } = require('chai') -const sinon = require('sinon') -const chai = require('chai') -const should = chai.should() -const { expect } = chai -const modulePath = '../../../app/js/BucketController.js' -const SandboxedModule = require('sandboxed-module') - -describe('BucketController', function() { - beforeEach(function() { - this.PersistorManager = { - sendStream: sinon.stub(), - copyFile: sinon.stub(), - deleteFile: sinon.stub() - } - - this.settings = { - s3: { - buckets: { - user_files: 'user_files' - } - }, - filestore: { - backend: 's3', - s3: { - secret: 'secret', - key: 'this_key' - } - } - } - - this.FileHandler = { - getFile: sinon.stub(), - deleteFile: sinon.stub(), - insertFile: sinon.stub(), - getDirectorySize: sinon.stub() - } - this.LocalFileWriter = {} - this.controller = SandboxedModule.require(modulePath, { - requires: { - './LocalFileWriter': this.LocalFileWriter, - './FileHandler': this.FileHandler, - './PersistorManager': this.PersistorManager, - 'settings-sharelatex': this.settings, - 'metrics-sharelatex': { - inc() {} - }, - 'logger-sharelatex': { - log() {}, - err() {} - } - } - }) - this.project_id = 'project_id' - this.file_id = 'file_id' - this.bucket = 'user_files' - this.key = `${this.project_id}/${this.file_id}` - this.req = { - query: {}, - params: { - bucket: this.bucket, - 0: this.key - }, - headers: {} - } - this.res = { setHeader() {} } - return (this.fileStream = {}) - }) - - return describe('getFile', function() { - it('should pipe the stream', function(done) { - this.FileHandler.getFile.callsArgWith(3, null, this.fileStream) - this.fileStream.pipe = res => { - res.should.equal(this.res) - return done() - } - return this.controller.getFile(this.req, this.res) - }) - - return it('should send a 500 if there is a problem', function(done) { - this.FileHandler.getFile.callsArgWith(3, 'error') - this.res.send = code => { - code.should.equal(500) - return done() - } - return this.controller.getFile(this.req, this.res) - }) - }) -}) From 3bf51cac67a5675352fd051e6378e6a3209310f9 Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Wed, 8 Jan 2020 09:17:30 +0000 Subject: [PATCH 72/73] Improve naming on internal '_client' method and use Map over object --- .../filestore/app/js/S3PersistorManager.js | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/services/filestore/app/js/S3PersistorManager.js b/services/filestore/app/js/S3PersistorManager.js index 00ed46379b..4ccc1642a4 100644 --- a/services/filestore/app/js/S3PersistorManager.js +++ b/services/filestore/app/js/S3PersistorManager.js @@ -64,7 +64,7 @@ async function sendStream(bucketName, key, readStream) { metrics.count('s3.egress', meteredStream.bytes) }) - const response = await _client(bucketName) + const response = await _getClientForBucket(bucketName) .upload({ Bucket: bucketName, Key: key, @@ -95,7 +95,7 @@ async function getFileStream(bucketName, key, opts) { } return new Promise((resolve, reject) => { - const stream = _client(bucketName) + const stream = _getClientForBucket(bucketName) .getObject(params) .createReadStream() @@ -120,7 +120,7 @@ async function deleteDirectory(bucketName, key) { let response try { - response = await _client(bucketName) + response = await _getClientForBucket(bucketName) .listObjects({ Bucket: bucketName, Prefix: key }) .promise() } catch (err) { @@ -135,7 +135,7 @@ async function deleteDirectory(bucketName, key) { const objects = response.Contents.map(item => ({ Key: item.Key })) if (objects.length) { try { - await _client(bucketName) + await _getClientForBucket(bucketName) .deleteObjects({ Bucket: bucketName, Delete: { @@ -157,7 +157,7 @@ async function deleteDirectory(bucketName, key) { async function getFileSize(bucketName, key) { try { - const response = await _client(bucketName) + const response = await _getClientForBucket(bucketName) .headObject({ Bucket: bucketName, Key: key }) .promise() return response.ContentLength @@ -173,7 +173,7 @@ async function getFileSize(bucketName, key) { async function deleteFile(bucketName, key) { try { - await _client(bucketName) + await _getClientForBucket(bucketName) .deleteObject({ Bucket: bucketName, Key: key }) .promise() } catch (err) { @@ -193,7 +193,7 @@ async function copyFile(bucketName, sourceKey, destKey) { CopySource: `${bucketName}/${sourceKey}` } try { - await _client(bucketName) + await _getClientForBucket(bucketName) .copyObject(params) .promise() } catch (err) { @@ -220,7 +220,7 @@ async function checkIfFileExists(bucketName, key) { async function directorySize(bucketName, key) { try { - const response = await _client(bucketName) + const response = await _getClientForBucket(bucketName) .listObjects({ Bucket: bucketName, Prefix: key }) .promise() @@ -249,9 +249,10 @@ function _wrapError(error, message, params, ErrorType) { } } -const _clients = {} +const _clients = new Map() +let _defaultClient -function _client(bucket) { +function _getClientForBucket(bucket) { if (_clients[bucket]) { return _clients[bucket] } @@ -261,19 +262,19 @@ function _client(bucket) { settings.filestore.s3.s3BucketCreds[bucket] ) { _clients[bucket] = new S3( - _clientOptions(settings.filestore.s3.s3BucketCreds[bucket]) + _buildClientOptions(settings.filestore.s3.s3BucketCreds[bucket]) ) return _clients[bucket] } // no specific credentials for the bucket - if (_clients.default) { - return _clients.default + if (_defaultClient) { + return _defaultClient } if (settings.filestore.s3.key) { - _clients.default = new S3(_clientOptions()) - return _clients.default + _defaultClient = new S3(_buildClientOptions()) + return _defaultClient } throw new SettingsError({ @@ -282,7 +283,7 @@ function _client(bucket) { }) } -function _clientOptions(bucketCredentials) { +function _buildClientOptions(bucketCredentials) { const options = {} if (bucketCredentials) { From d8470d4a59b36866072cdc9fe05840f06572deed Mon Sep 17 00:00:00 2001 From: Simon Detheridge Date: Wed, 8 Jan 2020 11:20:44 +0000 Subject: [PATCH 73/73] Use correct object path for s3BucketCreds --- services/filestore/app/js/S3PersistorManager.js | 6 +++--- services/filestore/test/unit/js/S3PersistorManagerTests.js | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/services/filestore/app/js/S3PersistorManager.js b/services/filestore/app/js/S3PersistorManager.js index 4ccc1642a4..5fb7040b24 100644 --- a/services/filestore/app/js/S3PersistorManager.js +++ b/services/filestore/app/js/S3PersistorManager.js @@ -258,11 +258,11 @@ function _getClientForBucket(bucket) { } if ( - settings.filestore.s3.s3BucketCreds && - settings.filestore.s3.s3BucketCreds[bucket] + settings.filestore.s3BucketCreds && + settings.filestore.s3BucketCreds[bucket] ) { _clients[bucket] = new S3( - _buildClientOptions(settings.filestore.s3.s3BucketCreds[bucket]) + _buildClientOptions(settings.filestore.s3BucketCreds[bucket]) ) return _clients[bucket] } diff --git a/services/filestore/test/unit/js/S3PersistorManagerTests.js b/services/filestore/test/unit/js/S3PersistorManagerTests.js index bdac7c8232..4e98aa01ff 100644 --- a/services/filestore/test/unit/js/S3PersistorManagerTests.js +++ b/services/filestore/test/unit/js/S3PersistorManagerTests.js @@ -195,8 +195,8 @@ describe('S3PersistorManagerTests', function() { } beforeEach(async function() { - settings.filestore.s3.s3BucketCreds = {} - settings.filestore.s3.s3BucketCreds[bucket] = { + settings.filestore.s3BucketCreds = {} + settings.filestore.s3BucketCreds[bucket] = { auth_key: alternativeKey, auth_secret: alternativeSecret }