Merge pull request #4344 from overleaf/revert-4304-ab-configurable-split-test

Revert "Configurable split tests system"

GitOrigin-RevId: dcaf79d074949c0c28b06515278a873c89b5aecb
This commit is contained in:
Alexandre Bourdin
2021-07-28 02:06:42 +00:00
committed by Copybot
parent eb31e6ebed
commit d28c1941c2
11 changed files with 3 additions and 2341 deletions
@@ -1,25 +0,0 @@
const SplitTestManager = require('./SplitTestManager')
const { SplitTest } = require('../../models/SplitTest')
const { CacheLoader } = require('cache-flow')
class SplitTestCache extends CacheLoader {
constructor() {
super('split-test', {
expirationTime: 60, // 1min in seconds
})
}
async load(name) {
return await SplitTestManager.getSplitTestByName(name)
}
serialize(value) {
return value.toObject()
}
deserialize(value) {
return new SplitTest(value)
}
}
module.exports = new SplitTestCache()
@@ -1,77 +0,0 @@
const SplitTestManager = require('./SplitTestManager')
async function getSplitTests(req, res, next) {
try {
const splitTests = await SplitTestManager.getSplitTests()
res.send(splitTests)
} catch (error) {
res.status(500).json({
error: `Error while fetching split tests list: ${error.message}`,
})
}
}
async function createSplitTest(req, res, next) {
const { name, configuration } = req.body
try {
const splitTest = await SplitTestManager.createSplitTest(
name,
configuration
)
res.send(splitTest)
} catch (error) {
res
.status(500)
.json({ error: `Error while creating split test: ${error.message}` })
}
}
async function updateSplitTest(req, res, next) {
const { name, configuration } = req.body
try {
const splitTest = await SplitTestManager.updateSplitTest(
name,
configuration
)
res.send(splitTest)
} catch (error) {
res
.status(500)
.json({ error: `Error while updating split test: ${error.message}` })
}
}
async function switchToNextPhase(req, res, next) {
const { name } = req.body
try {
const splitTest = await SplitTestManager.switchToNextPhase(name)
res.send(splitTest)
} catch (error) {
res.status(500).json({
error: `Error while switching split test to next phase: ${error.message}`,
})
}
}
async function revertToPreviousVersion(req, res, next) {
const { name, versionNumber } = req.body
try {
const splitTest = await SplitTestManager.revertToPreviousVersion(
name,
versionNumber
)
res.send(splitTest)
} catch (error) {
res.status(500).json({
error: `Error while reverting to previous version: ${error.message}`,
})
}
}
module.exports = {
getSplitTests,
createSplitTest,
updateSplitTest,
switchToNextPhase,
revertToPreviousVersion,
}
@@ -114,14 +114,8 @@ function _getPercentile(userId, splitTestId) {
}
module.exports = {
/**
* @deprecated: use SplitTestV2Handler.getAssignment instead
*/
getTestSegmentation: callbackify(getTestSegmentation),
promises: {
/**
* @deprecated: use SplitTestV2Handler.promises.getAssignment instead
*/
getTestSegmentation,
},
}
@@ -1,230 +0,0 @@
const { SplitTest } = require('../../models/SplitTest')
const OError = require('@overleaf/o-error')
const _ = require('lodash')
const ALPHA_PHASE = 'alpha'
const BETA_PHASE = 'beta'
const RELEASE_PHASE = 'release'
async function getSplitTests() {
try {
return await SplitTest.find().exec()
} catch (error) {
throw OError.tag(error, 'Failed to get split tests list')
}
}
async function getSplitTestByName(name) {
try {
return await SplitTest.findOne({ name }).exec()
} catch (error) {
throw OError.tag(error, 'Failed to get split test', { name })
}
}
async function createSplitTest(name, configuration) {
const stripedVariants = []
let stripeStart = 0
_checkNewVariantsConfiguration([], configuration.variants)
for (const variant of configuration.variants) {
stripedVariants.push({
name: variant.name,
active: variant.active,
rolloutPercent: variant.rolloutPercent,
rolloutStripes: [
{
start: stripeStart,
end: stripeStart + variant.rolloutPercent,
},
],
})
stripeStart += variant.rolloutPercent
}
const splitTest = new SplitTest({
name,
versions: [
{
versionNumber: 1,
phase: configuration.phase,
active: configuration.active,
variants: stripedVariants,
},
],
})
return _saveSplitTest(splitTest)
}
async function updateSplitTest(name, configuration) {
const splitTest = await getSplitTestByName(name)
if (splitTest) {
const lastVersion = splitTest.getCurrentVersion().toObject()
if (configuration.phase !== lastVersion.phase) {
throw new OError(
`Cannot update with different phase - use switchToNextPhase endpoint instead`
)
}
_checkNewVariantsConfiguration(lastVersion.variants, configuration.variants)
const updatedVariants = _updateVariantsWithNewConfiguration(
lastVersion.variants,
configuration.variants
)
splitTest.versions.push({
versionNumber: lastVersion.versionNumber + 1,
phase: configuration.phase,
active: configuration.active,
variants: updatedVariants,
})
return _saveSplitTest(splitTest)
} else {
throw new OError(`Cannot update split test '${name}': not found`)
}
}
async function switchToNextPhase(name) {
const splitTest = await getSplitTestByName(name)
if (splitTest) {
const lastVersionCopy = splitTest.getCurrentVersion().toObject()
lastVersionCopy.versionNumber++
if (lastVersionCopy.phase === ALPHA_PHASE) {
lastVersionCopy.phase = BETA_PHASE
} else if (lastVersionCopy.phase === BETA_PHASE) {
if (splitTest.forbidReleasePhase) {
throw new OError('Switch to release phase is disabled for this test')
}
lastVersionCopy.phase = RELEASE_PHASE
} else if (splitTest.phase === RELEASE_PHASE) {
throw new OError(
`Split test with ID '${name}' is already in the release phase`
)
}
for (const variant of lastVersionCopy.variants) {
variant.rolloutPercent = 0
variant.rolloutStripes = []
}
splitTest.versions.push(lastVersionCopy)
return _saveSplitTest(splitTest)
} else {
throw new OError(
`Cannot switch split test with ID '${name}' to next phase: not found`
)
}
}
async function revertToPreviousVersion(name, versionNumber) {
const splitTest = await getSplitTestByName(name)
if (splitTest) {
if (splitTest.versions.length <= 1) {
throw new OError(
`Cannot revert split test with ID '${name}' to previous version: split test must have at least 2 versions`
)
}
const previousVersion = splitTest.getVersion(versionNumber)
if (!previousVersion) {
throw new OError(
`Cannot revert split test with ID '${name}' to version number ${versionNumber}: version not found`
)
}
const lastVersion = splitTest.getCurrentVersion()
if (
lastVersion.phase === RELEASE_PHASE &&
previousVersion.phase !== RELEASE_PHASE
) {
splitTest.forbidReleasePhase = true
}
const previousVersionCopy = previousVersion.toObject()
previousVersionCopy.versionNumber = lastVersion.versionNumber + 1
splitTest.versions.push(previousVersionCopy)
return _saveSplitTest(splitTest)
} else {
throw new OError(
`Cannot revert split test with ID '${name}' to previous version: not found`
)
}
}
function _checkNewVariantsConfiguration(variants, newVariantsConfiguration) {
const totalRolloutPercentage = _getTotalRolloutPercentage(
newVariantsConfiguration
)
if (totalRolloutPercentage > 100) {
throw new OError(`Total variants rollout percentage cannot exceed 100`)
}
for (const variant of variants) {
const newVariantConfiguration = _.find(newVariantsConfiguration, {
name: variant.name,
})
if (!newVariantConfiguration) {
throw new OError(
`Variant defined in previous version as ${JSON.stringify(
variant
)} cannot be removed in new configuration: either set it inactive or create a new split test`
)
}
if (newVariantConfiguration.rolloutPercent < variant.rolloutPercent) {
throw new OError(
`Rollout percentage for variant defined in previous version as ${JSON.stringify(
variant
)} cannot be decreased: revert to a previous configuration instead`
)
}
}
}
function _updateVariantsWithNewConfiguration(
variants,
newVariantsConfiguration
) {
let totalRolloutPercentage = _getTotalRolloutPercentage(variants)
const variantsCopy = _.clone(variants)
for (const newVariantConfig of newVariantsConfiguration) {
const variant = _.find(variantsCopy, { name: newVariantConfig.name })
if (!variant) {
variantsCopy.push({
name: newVariantConfig.name,
active: newVariantConfig.active,
rolloutPercent: newVariantConfig.rolloutPercent,
rolloutStripes: [
{
start: totalRolloutPercentage,
end: totalRolloutPercentage + newVariantConfig.rolloutPercent,
},
],
})
totalRolloutPercentage += newVariantConfig.rolloutPercent
} else if (variant.rolloutPercent < newVariantConfig.rolloutPercent) {
const newStripeSize =
newVariantConfig.rolloutPercent - variant.rolloutPercent
variant.active = newVariantConfig.active
variant.rolloutPercent = newVariantConfig.rolloutPercent
variant.rolloutStripes.push({
start: totalRolloutPercentage,
end: totalRolloutPercentage + newStripeSize,
})
totalRolloutPercentage += newStripeSize
}
}
return variantsCopy
}
function _getTotalRolloutPercentage(variants) {
return _.sumBy(variants, 'rolloutPercent')
}
async function _saveSplitTest(splitTest) {
try {
return (await splitTest.save()).toObject()
} catch (error) {
throw OError.tag(error, 'Failed to save split test', {
splitTest: JSON.stringify(splitTest),
})
}
}
module.exports = {
getSplitTestByName,
getSplitTests,
createSplitTest,
updateSplitTest,
switchToNextPhase,
revertToPreviousVersion,
}
@@ -1,49 +0,0 @@
const SplitTestController = require('./SplitTestController')
const AuthorizationMiddleware = require('../Authorization/AuthorizationMiddleware')
const Features = require('../../infrastructure/Features')
module.exports = {
apply(webRouter) {
if (Features.hasFeature('saas')) {
webRouter.get(
'/admin/splitTests',
AuthorizationMiddleware.ensureUserIsSiteAdmin,
SplitTestController.getSplitTests
)
webRouter.post(
'/admin/createSplitTest',
AuthorizationMiddleware.ensureUserIsSiteAdmin,
SplitTestController.createSplitTest
)
webRouter.csrf.disableDefaultCsrfProtection('/admin/splitTest', 'PUT')
webRouter.post(
'/admin/updateSplitTest',
AuthorizationMiddleware.ensureUserIsSiteAdmin,
SplitTestController.updateSplitTest
)
webRouter.csrf.disableDefaultCsrfProtection('/admin/splitTest', 'POST')
webRouter.post(
'/admin/splitTest/switchToNextPhase',
AuthorizationMiddleware.ensureUserIsSiteAdmin,
SplitTestController.switchToNextPhase
)
webRouter.csrf.disableDefaultCsrfProtection(
'/admin/splitTest/switchToNextPhase',
'POST'
)
webRouter.post(
'/admin/splitTest/revertToPreviousVersion',
AuthorizationMiddleware.ensureUserIsSiteAdmin,
SplitTestController.revertToPreviousVersion
)
webRouter.csrf.disableDefaultCsrfProtection(
'/admin/splitTest/revertToPreviousVersion',
'POST'
)
}
},
}
@@ -1,177 +0,0 @@
const UserGetter = require('../User/UserGetter')
const UserUpdater = require('../User/UserUpdater')
const AnalyticsManager = require('../Analytics/AnalyticsManager')
const crypto = require('crypto')
const _ = require('lodash')
const { callbackify } = require('util')
const splitTestCache = require('./SplitTestCache')
const DEFAULT_VARIANT = 'default'
const ALPHA_PHASE = 'alpha'
const BETA_PHASE = 'beta'
/**
* Get the assignment of a user to a split test.
*
* @example
* // Assign user and record an event
*
* const assignment = await SplitTestV2Handler.getAssignment(userId, 'example-project')
* if (assignment.variant === 'awesome-new-version') {
* // execute my awesome change
* }
* else {
* // execute the default behaviour (control group)
* }
* // then record an event
* AnalyticsManager.recordEvent(userId, 'example-project-created', {
* projectId: project._id,
* ...assignment.analytics.segmentation
* })
*
* @param userId the user's ID
* @param splitTestName the unique name of the split test
* @param options {sync: boolean} - for test purposes only, to force the synchronous update of the user's profile
* @returns {Promise<{analytics: {segmentation: {}}, variant: string}|{analytics: {segmentation: {phase, splitTest, variant: string, versionNumber}}, variant: string}>}
*/
async function getAssignment(userId, splitTestName, options) {
const splitTest = await splitTestCache.get(splitTestName)
if (splitTest) {
const currentVersion = splitTest.getCurrentVersion()
if (currentVersion.active) {
const {
activeForUser,
selectedVariantName,
phase,
versionNumber,
} = await _getAssignmentMetadata(userId, splitTest)
if (activeForUser) {
const assignmentConfig = {
userId,
splitTestName,
variantName: selectedVariantName,
phase,
versionNumber,
}
if (options && options.sync === true) {
await _updateVariantAssignment(assignmentConfig)
} else {
_updateVariantAssignment(assignmentConfig)
}
return {
variant: selectedVariantName,
analytics: {
segmentation: {
splitTest: splitTestName,
variant: selectedVariantName,
phase,
versionNumber,
},
},
}
}
}
}
return {
variant: DEFAULT_VARIANT,
analytics: {
segmentation: {},
},
}
}
async function _getAssignmentMetadata(userId, splitTest) {
const currentVersion = splitTest.getCurrentVersion()
const phase = currentVersion.phase
if ([ALPHA_PHASE, BETA_PHASE].includes(phase)) {
const user = await _getUser(userId)
if (
(phase === ALPHA_PHASE && !(user && user.alphaProgram)) ||
(phase === BETA_PHASE && !(user && user.betaProgram))
) {
return {
activeForUser: false,
}
}
}
const percentile = _getPercentile(userId, splitTest.name, phase)
const selectedVariantName = _getVariantFromPercentile(
currentVersion.variants,
percentile
)
return {
activeForUser: true,
selectedVariantName: selectedVariantName || DEFAULT_VARIANT,
phase,
versionNumber: currentVersion.versionNumber,
}
}
function _getPercentile(userId, splitTestName, splitTestPhase) {
const hash = crypto
.createHash('md5')
.update(userId + splitTestName + splitTestPhase)
.digest('hex')
const hashPrefix = hash.substr(0, 8)
return Math.floor(
((parseInt(hashPrefix, 16) % 0xffffffff) / 0xffffffff) * 100
)
}
function _getVariantFromPercentile(variants, percentile) {
for (const variant of variants) {
for (const stripe of variant.rolloutStripes) {
if (percentile >= stripe.start && percentile < stripe.end) {
return variant.name
}
}
}
}
async function _updateVariantAssignment({
userId,
splitTestName,
phase,
versionNumber,
variantName,
}) {
const user = await _getUser(userId)
if (user) {
const assignedSplitTests = user.splitTests || []
const assignmentLog = assignedSplitTests[splitTestName] || []
const existingAssignment = _.find(assignmentLog, { versionNumber })
if (!existingAssignment) {
await UserUpdater.promises.updateUser(userId, {
$addToSet: {
[`splitTests.${splitTestName}`]: {
variantName,
versionNumber,
phase,
assignedAt: new Date(),
},
},
})
AnalyticsManager.setUserProperty(
userId,
`split-test-${splitTestName}-${versionNumber}`,
variantName
)
}
}
}
async function _getUser(id) {
return UserGetter.promises.getUser(id, {
splitTests: 1,
alphaProgram: 1,
betaProgram: 1,
})
}
module.exports = {
getAssignment: callbackify(getAssignment),
promises: {
getAssignment,
},
}
-111
View File
@@ -1,111 +0,0 @@
const mongoose = require('../infrastructure/Mongoose')
const { Schema } = mongoose
const _ = require('lodash')
const MIN_NAME_LENGTH = 3
const MAX_NAME_LENGTH = 200
const MIN_VARIANT_NAME_LENGTH = 3
const MAX_VARIANT_NAME_LENGTH = 255
const NAME_REGEX = /^[a-zA-Z0-9\-_]+$/
const RolloutPercentType = {
type: Number,
default: 0,
min: [0, 'Rollout percentage must be between 0 and 100, got {VALUE}'],
max: [100, 'Rollout percentage must be between 0 and 100, got {VALUE}'],
required: true,
}
const VariantSchema = new Schema(
{
name: {
type: String,
minLength: MIN_VARIANT_NAME_LENGTH,
maxLength: MAX_VARIANT_NAME_LENGTH,
required: true,
validate: {
validator: function (input) {
return input !== null && input !== 'default' && NAME_REGEX.test(input)
},
message: `invalid, cannot be 'default' and must match: ${NAME_REGEX}, got {VALUE}`,
},
},
active: {
type: Boolean,
default: true,
required: true,
},
rolloutPercent: RolloutPercentType,
rolloutStripes: [
{
start: RolloutPercentType,
end: RolloutPercentType,
},
],
},
{ _id: false }
)
const VersionSchema = new Schema(
{
versionNumber: {
type: Number,
default: 1,
min: [1, 'must be 1 or higher, got {VALUE}'],
required: true,
},
phase: {
type: String,
default: 'alpha',
enum: ['alpha', 'beta', 'release'],
required: true,
},
active: {
type: Boolean,
default: true,
required: true,
},
variants: [VariantSchema],
},
{ _id: false }
)
const SplitTestSchema = new Schema({
name: {
type: String,
minLength: MIN_NAME_LENGTH,
maxlength: MAX_NAME_LENGTH,
required: true,
unique: true,
validate: {
validator: function (input) {
return input !== null && NAME_REGEX.test(input)
},
message: `invalid, must match: ${NAME_REGEX}`,
},
},
versions: [VersionSchema],
forbidReleasePhase: {
type: Boolean,
required: false,
},
})
SplitTestSchema.methods.getCurrentVersion = function () {
if (this.versions && this.versions.length > 0) {
return _.maxBy(this.versions, 'versionNumber')
} else {
return undefined
}
}
SplitTestSchema.methods.getVersion = function (versionNumber) {
return _.find(this.versions || [], {
versionNumber,
})
}
module.exports = {
SplitTest: mongoose.model('SplitTest', SplitTestSchema),
SplitTestSchema,
}
-2
View File
@@ -49,7 +49,6 @@ const InstitutionsController = require('./Features/Institutions/InstitutionsCont
const UserMembershipRouter = require('./Features/UserMembership/UserMembershipRouter')
const SystemMessageController = require('./Features/SystemMessages/SystemMessageController')
const AnalyticsRegistrationSourceMiddleware = require('./Features/Analytics/AnalyticsRegistrationSourceMiddleware')
const SplitTestRouter = require('./Features/SplitTests/SplitTestRouter')
const { Joi, validate } = require('./infrastructure/Validation')
const {
renderUnsupportedBrowserPage,
@@ -108,7 +107,6 @@ function initialize(webRouter, privateApiRouter, publicApiRouter) {
LinkedFilesRouter.apply(webRouter, privateApiRouter, publicApiRouter)
TemplatesRouter.apply(webRouter)
UserMembershipRouter.apply(webRouter)
SplitTestRouter.apply(webRouter)
Modules.applyRouter(webRouter, privateApiRouter, publicApiRouter)