alois/Verso
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #6332 from overleaf/ab-split-test-param-overrides

Browse Source 
Split tests query param overrides

GitOrigin-RevId: 8112710d057ddc22cebf37a619dfc969be57b6cc
This commit is contained in:
Alexandre Bourdin
2022-01-25 09:03:04 +00:00
committed by Copybot
parent bbac46156b
commit 1b954fa720
10 changed files with 313 additions and 819 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/web/app/src/Features/Project/ProjectController.js
+8 -11
View File
@@ -37,7 +37,7 @@ const BrandVariationsHandler = require('../BrandVariations/BrandVariationsHandle
const UserController = require('../User/UserController')
const AnalyticsManager = require('../Analytics/AnalyticsManager')
const Modules = require('../../infrastructure/Modules')
const SplitTestV2Handler = require('../SplitTests/SplitTestV2Handler')
const SplitTestHandler = require('../SplitTests/SplitTestHandler')
const { getNewLogsUIVariantForUser } = require('../Helpers/NewLogsUI')
const FeaturesUpdater = require('../Subscription/FeaturesUpdater')
const SpellingHandler = require('../Spelling/SpellingHandler')
@@ -607,10 +607,7 @@ const ProjectController = {
}
// null test targeting logged in users
SplitTestV2Handler.promises.getAssignmentForSession(
req.session,
'null-test-dashboard'
)
SplitTestHandler.promises.getAssignment(req, 'null-test-dashboard')
res.render('project/list', viewModel)
timer.done()
@@ -737,9 +734,9 @@ const ProjectController = {
TpdsProjectFlusher.flushProjectToTpdsIfNeeded(projectId, cb)
},
sharingModalSplitTest(cb) {
SplitTestV2Handler.assignInLocalsContextForSession(
SplitTestHandler.assignInLocalsContext(
req,
res,
req.session,
'project-share-modal-paywall',
{},
() => {
@@ -750,9 +747,9 @@ const ProjectController = {
},
sharingModalNullTest(cb) {
// null test targeting logged in users, for front-end side
SplitTestV2Handler.assignInLocalsContextForSession(
SplitTestHandler.assignInLocalsContext(
req,
res,
req.session,
'null-test-share-modal',
{},
() => {
@@ -762,8 +759,8 @@ const ProjectController = {
)
},
newPdfPreviewAssignment(cb) {
SplitTestV2Handler.getAssignmentForSession(
req.session,
SplitTestHandler.getAssignment(
req,
'react-pdf-preview-rollout',
{},
(error, assignment) => {
services/web/app/src/Features/SplitTests/SplitTestHandler.js
+259 -102
View File
@@ -1,127 +1,284 @@
const UserGetter = require('../User/UserGetter')
const UserUpdater = require('../User/UserUpdater')
const AnalyticsManager = require('../Analytics/AnalyticsManager')
const Settings = require('@overleaf/settings')
const _ = require('lodash')
const LocalsHelper = require('./LocalsHelper')
const crypto = require('crypto')
const OError = require('@overleaf/o-error')
const _ = require('lodash')
const { callbackify } = require('util')
const splitTestCache = require('./SplitTestCache')
const duplicateSplitTest = _.findKey(
_.groupBy(Settings.splitTests, 'id'),
group => {
return group.length > 1
const DEFAULT_VARIANT = 'default'
const ALPHA_PHASE = 'alpha'
const BETA_PHASE = 'beta'
const DEFAULT_ASSIGNMENT = {
variant: DEFAULT_VARIANT,
analytics: {
segmentation: {},
},
}
/**
* Get the assignment of a user to a split test by their session.
*
* @example
* // Assign user and record an event
*
* const assignment = await SplitTestHandler.getAssignment(req.session, 'example-project')
* if (assignment.variant === 'awesome-new-version') {
* // execute my awesome change
* }
* else {
* // execute the default behaviour (control group)
* }
* // then record an event
* AnalyticsManager.recordEventForSession(req.session, 'example-project-created', {
* projectId: project._id,
* ...assignment.analytics.segmentation
* })
*
* @param req the request
* @param splitTestName the unique name of the split test
* @param options {Object<sync: boolean>} - for test purposes only, to force the synchronous update of the user's profile
* @returns {Promise<{variant: string, analytics: {segmentation: {splitTest: string, variant: string, phase: string, versionNumber: number}|{}}}>}
*/
async function getAssignment(req, splitTestName, options) {
const query = req.query || {}
if (query[splitTestName]) {
return {
variant: query[splitTestName],
analytics: {
segmentation: {},
},
}
}
)
if (duplicateSplitTest) {
throw new OError(
`Split test IDs must be unique: ${duplicateSplitTest} is defined at least twice`
const { userId, analyticsId } = AnalyticsManager.getIdsFromSession(
req.session
)
return _getAssignment(
analyticsId,
userId,
req.session,
splitTestName,
options
)
}
const ACTIVE_SPLIT_TESTS = []
for (const splitTest of Settings.splitTests) {
for (const variant of splitTest.variants) {
if (variant.id === 'default') {
throw new OError(
`Split test variant ID cannot be 'default' (reserved value), defined in split test ${JSON.stringify(
splitTest
)}`
/**
* Get the assignment of a user to a split test by their session and stores it in the locals context.
*
* @param req the request
* @param res the Express response object
* @param splitTestName the unique name of the split test
* @param options {Object<sync: boolean>} - for test purposes only, to force the synchronous update of the user's profile
* @returns {Promise<void>}
*/
async function assignInLocalsContext(req, res, splitTestName, options) {
const assignment = await getAssignment(req, splitTestName, options)
LocalsHelper.setSplitTestVariant(
res.locals,
splitTestName,
assignment.variant
)
}
async function _getAssignment(
analyticsId,
userId,
session,
splitTestName,
options
) {
if (!analyticsId && !userId) {
return DEFAULT_ASSIGNMENT
}
const splitTest = await splitTestCache.get(splitTestName)
if (splitTest) {
const currentVersion = splitTest.getCurrentVersion()
const cachedVariant = _getCachedVariantFromSession(
session,
splitTest.name,
currentVersion
)
if (currentVersion.active) {
if (cachedVariant) {
return _makeAssignment(splitTest, cachedVariant, currentVersion)
}
const { activeForUser, selectedVariantName, phase, versionNumber } =
await _getAssignmentMetadata(analyticsId, userId, splitTest)
if (activeForUser) {
const assignmentConfig = {
userId,
analyticsId,
session,
splitTestName,
variantName: selectedVariantName,
phase,
versionNumber,
}
if (options && options.sync === true) {
await _updateVariantAssignment(assignmentConfig)
} else {
_updateVariantAssignment(assignmentConfig)
}
return _makeAssignment(splitTest, selectedVariantName, currentVersion)
}
}
}
return DEFAULT_ASSIGNMENT
}
async function _getAssignmentMetadata(analyticsId, userId, splitTest) {
const currentVersion = splitTest.getCurrentVersion()
const phase = currentVersion.phase
if ([ALPHA_PHASE, BETA_PHASE].includes(phase)) {
if (userId) {
const user = await _getUser(userId)
if (
(phase === ALPHA_PHASE && !(user && user.alphaProgram)) ||
(phase === BETA_PHASE && !(user && user.betaProgram))
) {
return {
activeForUser: false,
}
}
} else {
return {
activeForUser: false,
}
}
}
const percentile = _getPercentile(analyticsId, splitTest.name, phase)
const selectedVariantName = _getVariantFromPercentile(
currentVersion.variants,
percentile
)
return {
activeForUser: true,
selectedVariantName: selectedVariantName || DEFAULT_VARIANT,
phase,
versionNumber: currentVersion.versionNumber,
}
}
function _getPercentile(analyticsId, splitTestName, splitTestPhase) {
const hash = crypto
.createHash('md5')
.update(analyticsId + splitTestName + splitTestPhase)
.digest('hex')
const hashPrefix = hash.substr(0, 8)
return Math.floor(
((parseInt(hashPrefix, 16) % 0xffffffff) / 0xffffffff) * 100
)
}
function _getVariantFromPercentile(variants, percentile) {
for (const variant of variants) {
for (const stripe of variant.rolloutStripes) {
if (percentile >= stripe.start && percentile < stripe.end) {
return variant.name
}
}
}
}
async function _updateVariantAssignment({
userId,
analyticsId,
session,
splitTestName,
phase,
versionNumber,
variantName,
}) {
const persistedAssignment = {
variantName,
versionNumber,
phase,
assignedAt: new Date(),
}
// if the user is logged in
if (userId) {
const user = await _getUser(userId)
if (user) {
const assignedSplitTests = user.splitTests || []
const assignmentLog = assignedSplitTests[splitTestName] || []
const existingAssignment = _.find(assignmentLog, { versionNumber })
if (!existingAssignment) {
await UserUpdater.promises.updateUser(userId, {
$addToSet: {
[`splitTests.${splitTestName}`]: persistedAssignment,
},
})
AnalyticsManager.setUserPropertyForAnalyticsId(
analyticsId,
`split-test-${splitTestName}-${versionNumber}`,
variantName
)
}
}
}
// otherwise this is an anonymous user, we store assignments in session to persist them on registration
else if (session) {
if (!session.splitTests) {
session.splitTests = {}
}
if (!session.splitTests[splitTestName]) {
session.splitTests[splitTestName] = []
}
const existingAssignment = _.find(session.splitTests[splitTestName], {
versionNumber,
})
if (!existingAssignment) {
session.splitTests[splitTestName].push(persistedAssignment)
AnalyticsManager.setUserPropertyForAnalyticsId(
analyticsId,
`split-test-${splitTestName}-${versionNumber}`,
variantName
)
}
}
const totalVariantsRolloutPercent = _.sumBy(
splitTest.variants,
'rolloutPercent'
)
if (splitTest.active) {
if (totalVariantsRolloutPercent > 100) {
for (const variant of splitTest.variants) {
variant.rolloutPercent =
(variant.rolloutPercent * 100) / totalVariantsRolloutPercent
}
}
if (totalVariantsRolloutPercent > 0) {
ACTIVE_SPLIT_TESTS.push(splitTest)
}
}
}
async function getTestSegmentation(userId, splitTestId) {
const splitTest = _.find(ACTIVE_SPLIT_TESTS, ['id', splitTestId])
if (splitTest) {
const alreadyAssignedVariant = await getAlreadyAssignedVariant(
userId,
splitTestId
)
if (alreadyAssignedVariant) {
return {
enabled: true,
variant: alreadyAssignedVariant,
}
} else {
const variant = await assignUserToVariant(userId, splitTest)
return {
enabled: true,
variant,
}
}
}
function _makeAssignment(splitTest, variant, currentVersion) {
return {
enabled: false,
}
}
async function getAlreadyAssignedVariant(userId, splitTestId) {
const user = await UserGetter.promises.getUser(userId, { splitTests: 1 })
if (user && user.splitTests) {
return user.splitTests[splitTestId]
}
return undefined
}
async function assignUserToVariant(userId, splitTest) {
let userIdAsPercentile = await _getPercentile(userId, splitTest.id)
let selectedVariant = 'default'
for (const variant of splitTest.variants) {
if (userIdAsPercentile < variant.rolloutPercent) {
selectedVariant = variant.id
break
} else {
userIdAsPercentile -= variant.rolloutPercent
}
}
await UserUpdater.promises.updateUser(userId, {
$set: {
[`splitTests.${splitTest.id}`]: selectedVariant,
variant,
analytics: {
segmentation: {
splitTest: splitTest.name,
variant,
phase: currentVersion.phase,
versionNumber: currentVersion.versionNumber,
},
},
})
AnalyticsManager.setUserPropertyForUser(
userId,
`split-test-${splitTest.id}`,
selectedVariant
)
return selectedVariant
}
}
function _getPercentile(userId, splitTestId) {
const hash = crypto
.createHash('md5')
.update(userId + splitTestId)
.digest('hex')
const hashPrefix = hash.substr(0, 8)
return Math.floor((parseInt(hashPrefix, 16) / 0xffffffff) * 100)
function _getCachedVariantFromSession(session, splitTestName, currentVersion) {
if (!session.cachedSplitTestAssignments) {
session.cachedSplitTestAssignments = {}
return
}
const cacheKey = `${splitTestName}-${currentVersion.versionNumber}`
if (currentVersion.active) {
return session.cachedSplitTestAssignments[cacheKey]
} else {
delete session.cachedSplitTestAssignments[cacheKey]
}
}
async function _getUser(id) {
return UserGetter.promises.getUser(id, {
splitTests: 1,
alphaProgram: 1,
betaProgram: 1,
})
}
module.exports = {
/**
* @deprecated: use SplitTestV2Handler.getAssignment instead
*/
getTestSegmentation: callbackify(getTestSegmentation),
getAssignment: callbackify(getAssignment),
assignInLocalsContext: callbackify(assignInLocalsContext),
promises: {
/**
* @deprecated: use SplitTestV2Handler.promises.getAssignment instead
*/
getTestSegmentation,
getAssignment,
assignInLocalsContext,
},
}
services/web/app/src/Features/SplitTests/SplitTestMiddleware.js
+6 -43
View File
@@ -1,27 +1,15 @@
const SplitTestV2Handler = require('./SplitTestV2Handler')
const SplitTestCache = require('./SplitTestCache')
const LocalsHelper = require('./LocalsHelper')
const SplitTestHandler = require('./SplitTestHandler')
const logger = require('@overleaf/logger')
function loadAssignmentsInLocals(splitTestNames) {
return async function (req, res, next) {
try {
if (!req.session.cachedSplitTestAssignments) {
req.session.cachedSplitTestAssignments = {}
}
for (const splitTestName of splitTestNames) {
if (req.query[splitTestName]) {
LocalsHelper.setSplitTestVariant(
res.locals,
splitTestName,
req.query[splitTestName]
)
} else {
const splitTest = await SplitTestCache.get(splitTestName)
if (splitTest) {
await _loadAssignmentInLocals(splitTest, req.session, res.locals)
}
}
await SplitTestHandler.promises.assignInLocalsContext(
req,
res,
splitTestName
)
}
} catch (error) {
logger.error(
@@ -33,31 +21,6 @@ function loadAssignmentsInLocals(splitTestNames) {
}
}
async function _loadAssignmentInLocals(splitTest, session, locals) {
const currentVersion = splitTest.getCurrentVersion()
const cacheKey = `${splitTest.name}-${currentVersion.versionNumber}`
if (currentVersion.active) {
const cachedVariant = session.cachedSplitTestAssignments[cacheKey]
if (cachedVariant) {
LocalsHelper.setSplitTestVariant(locals, splitTest.name, cachedVariant)
} else {
const assignment =
await SplitTestV2Handler.promises.getAssignmentForSession(
session,
splitTest.name
)
session.cachedSplitTestAssignments[cacheKey] = assignment.variant
LocalsHelper.setSplitTestVariant(
locals,
splitTest.name,
assignment.variant
)
}
} else {
delete session.cachedSplitTestAssignments[cacheKey]
}
}
module.exports = {
loadAssignmentsInLocals,
}
services/web/app/src/Features/SplitTests/SplitTestV2Handler.js
-303
View File
@@ -1,303 +0,0 @@
const UserGetter = require('../User/UserGetter')
const UserUpdater = require('../User/UserUpdater')
const AnalyticsManager = require('../Analytics/AnalyticsManager')
const UserAnalyticsIdCache = require('../Analytics/UserAnalyticsIdCache')
const LocalsHelper = require('./LocalsHelper')
const crypto = require('crypto')
const _ = require('lodash')
const { callbackify } = require('util')
const splitTestCache = require('./SplitTestCache')
const DEFAULT_VARIANT = 'default'
const ALPHA_PHASE = 'alpha'
const BETA_PHASE = 'beta'
const DEFAULT_ASSIGNMENT = {
variant: DEFAULT_VARIANT,
analytics: {
segmentation: {},
},
}
/**
* Get the assignment of a user to a split test.
*
* @example
* // Assign user and record an event
*
* const assignment = await SplitTestV2Handler.getAssignment(userId, 'example-project')
* if (assignment.variant === 'awesome-new-version') {
* // execute my awesome change
* }
* else {
* // execute the default behaviour (control group)
* }
* // then record an event
* AnalyticsManager.recordEventForUser(userId, 'example-project-created', {
* projectId: project._id,
* ...assignment.analytics.segmentation
* })
*
* @param userId the user's ID
* @param splitTestName the unique name of the split test
* @param options {Object<sync: boolean>} - for test purposes only, to force the synchronous update of the user's profile
* @returns {Promise<{variant: string, analytics: {segmentation: {splitTest: string, variant: string, phase: string, versionNumber: number}|{}}}>}
*/
async function getAssignment(userId, splitTestName, options) {
if (!userId) {
return DEFAULT_ASSIGNMENT
}
const analyticsId = await UserAnalyticsIdCache.get(userId)
return _getAssignment(analyticsId, userId, undefined, splitTestName, options)
}
/**
* Get the assignment of a user to a split test by their session.
*
* @example
* // Assign user and record an event
*
* const assignment = await SplitTestV2Handler.getAssignment(req.session, 'example-project')
* if (assignment.variant === 'awesome-new-version') {
* // execute my awesome change
* }
* else {
* // execute the default behaviour (control group)
* }
* // then record an event
* AnalyticsManager.recordEventForSession(req.session, 'example-project-created', {
* projectId: project._id,
* ...assignment.analytics.segmentation
* })
*
* @param session the request session
* @param splitTestName the unique name of the split test
* @param options {Object<sync: boolean>} - for test purposes only, to force the synchronous update of the user's profile
* @returns {Promise<{variant: string, analytics: {segmentation: {splitTest: string, variant: string, phase: string, versionNumber: number}|{}}}>}
*/
async function getAssignmentForSession(session, splitTestName, options) {
const { userId, analyticsId } = AnalyticsManager.getIdsFromSession(session)
return _getAssignment(analyticsId, userId, session, splitTestName, options)
}
/**
* Get the assignment of a user to a split test by their ID and stores it in the locals context.
*
* @param res the Express response object
* @param userId the user ID
* @param splitTestName the unique name of the split test
* @param options {Object<sync: boolean>} - for test purposes only, to force the synchronous update of the user's profile
* @returns {Promise<void>}
*/
async function assignInLocalsContext(res, userId, splitTestName, options) {
const assignment = await getAssignment(userId, splitTestName, options)
LocalsHelper.setSplitTestVariant(
res.locals,
splitTestName,
assignment.variant
)
}
/**
* Get the assignment of a user to a split test by their session and stores it in the locals context.
*
* @param res the Express response object
* @param session the request session
* @param splitTestName the unique name of the split test
* @param options {Object<sync: boolean>} - for test purposes only, to force the synchronous update of the user's profile
* @returns {Promise<void>}
*/
async function assignInLocalsContextForSession(
res,
session,
splitTestName,
options
) {
const assignment = await getAssignmentForSession(
session,
splitTestName,
options
)
LocalsHelper.setSplitTestVariant(
res.locals,
splitTestName,
assignment.variant
)
}
async function _getAssignment(
analyticsId,
userId,
session,
splitTestName,
options
) {
if (!analyticsId && !userId) {
return DEFAULT_ASSIGNMENT
}
const splitTest = await splitTestCache.get(splitTestName)
if (splitTest) {
const currentVersion = splitTest.getCurrentVersion()
if (currentVersion.active) {
const { activeForUser, selectedVariantName, phase, versionNumber } =
await _getAssignmentMetadata(analyticsId, userId, splitTest)
if (activeForUser) {
const assignmentConfig = {
userId,
analyticsId,
session,
splitTestName,
variantName: selectedVariantName,
phase,
versionNumber,
}
if (options && options.sync === true) {
await _updateVariantAssignment(assignmentConfig)
} else {
_updateVariantAssignment(assignmentConfig)
}
return {
variant: selectedVariantName,
analytics: {
segmentation: {
splitTest: splitTestName,
variant: selectedVariantName,
phase,
versionNumber,
},
},
}
}
}
}
return DEFAULT_ASSIGNMENT
}
async function _getAssignmentMetadata(analyticsId, userId, splitTest) {
const currentVersion = splitTest.getCurrentVersion()
const phase = currentVersion.phase
if ([ALPHA_PHASE, BETA_PHASE].includes(phase)) {
if (userId) {
const user = await _getUser(userId)
if (
(phase === ALPHA_PHASE && !(user && user.alphaProgram)) ||
(phase === BETA_PHASE && !(user && user.betaProgram))
) {
return {
activeForUser: false,
}
}
} else {
return {
activeForUser: false,
}
}
}
const percentile = _getPercentile(analyticsId, splitTest.name, phase)
const selectedVariantName = _getVariantFromPercentile(
currentVersion.variants,
percentile
)
return {
activeForUser: true,
selectedVariantName: selectedVariantName || DEFAULT_VARIANT,
phase,
versionNumber: currentVersion.versionNumber,
}
}
function _getPercentile(analyticsId, splitTestName, splitTestPhase) {
const hash = crypto
.createHash('md5')
.update(analyticsId + splitTestName + splitTestPhase)
.digest('hex')
const hashPrefix = hash.substr(0, 8)
return Math.floor(
((parseInt(hashPrefix, 16) % 0xffffffff) / 0xffffffff) * 100
)
}
function _getVariantFromPercentile(variants, percentile) {
for (const variant of variants) {
for (const stripe of variant.rolloutStripes) {
if (percentile >= stripe.start && percentile < stripe.end) {
return variant.name
}
}
}
}
async function _updateVariantAssignment({
userId,
analyticsId,
session,
splitTestName,
phase,
versionNumber,
variantName,
}) {
const persistedAssignment = {
variantName,
versionNumber,
phase,
assignedAt: new Date(),
}
if (userId) {
const user = await _getUser(userId)
if (user) {
const assignedSplitTests = user.splitTests || []
const assignmentLog = assignedSplitTests[splitTestName] || []
const existingAssignment = _.find(assignmentLog, { versionNumber })
if (!existingAssignment) {
await UserUpdater.promises.updateUser(userId, {
$addToSet: {
[`splitTests.${splitTestName}`]: persistedAssignment,
},
})
AnalyticsManager.setUserPropertyForAnalyticsId(
analyticsId,
`split-test-${splitTestName}-${versionNumber}`,
variantName
)
}
}
} else if (session) {
if (!session.splitTests) {
session.splitTests = {}
}
if (!session.splitTests[splitTestName]) {
session.splitTests[splitTestName] = []
}
const existingAssignment = _.find(session.splitTests[splitTestName], {
versionNumber,
})
if (!existingAssignment) {
session.splitTests[splitTestName].push(persistedAssignment)
AnalyticsManager.setUserPropertyForAnalyticsId(
analyticsId,
`split-test-${splitTestName}-${versionNumber}`,
variantName
)
}
}
}
async function _getUser(id) {
return UserGetter.promises.getUser(id, {
splitTests: 1,
alphaProgram: 1,
betaProgram: 1,
})
}
module.exports = {
getAssignment: callbackify(getAssignment),
getAssignmentForSession: callbackify(getAssignmentForSession),
assignInLocalsContext: callbackify(assignInLocalsContext),
assignInLocalsContextForSession: callbackify(assignInLocalsContextForSession),
promises: {
getAssignment,
getAssignmentForSession,
assignInLocalsContext,
assignInLocalsContextForSession,
},
}
services/web/app/src/Features/Subscription/RecurlyEventHandler.js
+2 -2
View File
@@ -1,5 +1,5 @@
const AnalyticsManager = require('../Analytics/AnalyticsManager')
const SplitTestV2Handler = require('../SplitTests/SplitTestV2Handler')
const SplitTestHandler = require('../SplitTests/SplitTestHandler')
const SubscriptionEmailHandler = require('./SubscriptionEmailHandler')
function sendRecurlyAnalyticsEvent(event, eventData) {
@@ -63,7 +63,7 @@ async function sendSubscriptionStartedEvent(eventData) {
// send the trial onboarding email
if (isTrial) {
const assignment = await SplitTestV2Handler.promises.getAssignment(
const assignment = await SplitTestHandler.promises.getAssignment(
userId,
'trial-onboarding-email'
)