[migration scripts] detect account drift between Recurly and Stripe (#32315)

* add a script to detect drift between recurly and stripe
* only flag when Recurly updated since subscription created in Stripe
* add --comparison-date option instead of stripe subscription created time
* reduce API calls by checking Recurly update time earlier

GitOrigin-RevId: f0ecb5644f7445603da020a46e32622fcd2eebf4
This commit is contained in:
Kristina
2026-03-24 10:38:09 +01:00
committed by Copybot
parent 205573c01a
commit fe7dad5644
5 changed files with 2027 additions and 290 deletions
@@ -1116,8 +1116,9 @@ export function areStripeAndRecurlyCardDetailsEqual(
recurlyPaymentMethod
) {
if (
!stripePaymentMethod ||
stripePaymentMethod.type !== 'card' ||
!stripePaymentMethod?.card ||
!stripePaymentMethod.card ||
!recurlyPaymentMethod?.lastFour
) {
return false
@@ -1129,3 +1130,461 @@ export function areStripeAndRecurlyCardDetailsEqual(
stripePaymentMethod.card.exp_year === recurlyPaymentMethod.expYear
)
}
// =============================================================================
// ADDRESS / STRING COMPARISON HELPERS
// =============================================================================
export function normalizeComparableString(value) {
if (value == null) return ''
return String(value).trim()
}
export function hasAnyAddressValue(address) {
if (!address || typeof address !== 'object') return false
return Object.values(address).some(v => normalizeComparableString(v) !== '')
}
export function ccEmailsToArray(ccEmails) {
if (ccEmails == null || ccEmails === undefined) return []
const normalisedEmails = String(ccEmails)
.split(/[\s,;]+/)
.filter(Boolean)
return [...new Set(normalisedEmails)]
}
export function normalizeTaxIdValue(taxId) {
const value = normalizeComparableString(taxId)
return value.replace(/[.\-_\s:/]/g, '').toLowerCase()
}
// =============================================================================
// CUSTOM FIELDS
// =============================================================================
export const RECURLY_CUSTOM_FIELD_NAMES = [
'channel',
'Industry',
'ol_sales_person',
'MigratedfromFreeAgent',
]
/**
* Extract Recurly custom fields as Stripe metadata.
*
* Expects an account object with a `customFields` array (Recurly npm SDK format)
* where each element has `name` and `value` properties.
*
* @param {object} account - Recurly account object (npm SDK format)
* @returns {{ metadata: Record<string, string>, counts: Record<string, number> }}
*/
export function extractRecurlyCustomFieldMetadata(account) {
const customFields = account?.customFields
if (!Array.isArray(customFields)) {
throw new Error(
'Unexpected Recurly response: account.customFields is missing or not an array'
)
}
/** @type {Record<string, string>} */
const metadata = {}
const counts = {
channel: 0,
Industry: 0,
ol_sales_person: 0,
MigratedfromFreeAgent: 0,
noCustomFields: 0,
}
if (customFields.length === 0) {
counts.noCustomFields = 1
return { metadata, counts }
}
for (const field of customFields) {
const name = field?.name?.trim()
if (!RECURLY_CUSTOM_FIELD_NAMES.includes(name)) continue
const rawValue = field?.value
if (rawValue == null) continue
const value = String(rawValue).trim()
if (!value) continue
metadata[name] = value
counts[name] = 1
}
return { metadata, counts }
}
function normalizeComparableAddress(address) {
if (!address || typeof address !== 'object') return null
const norm = {
line1: normalizeComparableString(address.line1),
line2: normalizeComparableString(address.line2),
city: normalizeComparableString(address.city),
state: normalizeComparableString(address.state),
postal_code: normalizeComparableString(address.postal_code),
country: normalizeComparableString(address.country).toUpperCase(),
}
return hasAnyAddressValue(norm) ? norm : null
}
export function addressesEqual(a, b) {
const na = normalizeComparableAddress(a)
const nb = normalizeComparableAddress(b)
if (!na && !nb) return true
if (!na || !nb) return false
return (
na.line1 === nb.line1 &&
na.line2 === nb.line2 &&
na.city === nb.city &&
na.state === nb.state &&
na.postal_code === nb.postal_code &&
na.country === nb.country
)
}
// =============================================================================
// CUSTOMER IDENTITY RESOLUTION
// =============================================================================
/**
* Resolve customer name, address, company, and VAT number from Recurly account data.
*
* For most customers, billing info and account info agree (or only one source is set),
* and the standard coalesce logic applies (billing info preferred, account as fallback).
*
* When any field has conflicting values across both sources, `fetchCollectionMethod`
* is called to determine which source wins:
*
* - automatic (web sales): billing info is used for the Stripe customer record.
* - manual (manual billing): account info is used for the Stripe customer record,
* and billing info is returned separately to be copied to the payment method's
* billing_details.
*
* @param {object} account - Recurly account object
* @param {() => Promise<string|null>} fetchCollectionMethod - Async callback that returns the
* subscription's collection method ('automatic' or 'manual') when a conflict is detected.
* @returns {Promise<{
* name: string|null,
* address: import('stripe').Stripe.AddressParam|null,
* companyName: string|null,
* vatNumber: string|null,
* collectionMethod: string|null,
* billingInfoForPaymentMethod: object|null
* }>}
*/
export async function resolveCustomerIdentity(account, fetchCollectionMethod) {
const billingName = extractNameFromBillingInfo(account)
const accountName = extractNameFromAccount(account)
const nameConflict =
billingName !== null && accountName !== null && billingName !== accountName
const billingAddress = normalizeRecurlyAddressToStripe(
account.billingInfo?.address
)
const accountAddress = normalizeRecurlyAddressToStripe(account?.address)
const addressConflict =
billingAddress !== null &&
accountAddress !== null &&
!addressesEqual(billingAddress, accountAddress)
const billingCompany = account.billingInfo?.company?.trim() || null
const accountCompany = account.company?.trim() || null
const companyConflict =
billingCompany !== null &&
accountCompany !== null &&
billingCompany !== accountCompany
const billingVat = account.billingInfo?.vatNumber?.trim() || null
const accountVat = account?.vatNumber?.trim() || null
const vatConflict =
billingVat !== null && accountVat !== null && billingVat !== accountVat
const hasConflict =
nameConflict || addressConflict || companyConflict || vatConflict
let name,
address,
companyName,
vatNumber,
collectionMethod,
billingInfoForPaymentMethod
if (!hasConflict) {
name = billingName ?? accountName
address = billingAddress ?? accountAddress
companyName = billingCompany ?? accountCompany
vatNumber = billingVat ?? accountVat
collectionMethod = null
billingInfoForPaymentMethod = null
} else {
collectionMethod = await fetchCollectionMethod()
if (!collectionMethod) {
throw new Error(
'Conflict between billing info and account fields, but no subscription found to determine collection method'
)
}
if (collectionMethod === 'automatic') {
name = billingName ?? accountName
address = billingAddress ?? accountAddress
companyName = billingCompany ?? accountCompany
vatNumber = billingVat ?? accountVat
billingInfoForPaymentMethod = null
} else if (collectionMethod === 'manual') {
name = accountName ?? billingName
address = accountAddress ?? billingAddress
companyName = accountCompany ?? billingCompany
vatNumber = accountVat ?? billingVat
billingInfoForPaymentMethod = account.billingInfo
} else {
throw new Error(`Unexpected collectionMethod: ${collectionMethod}`)
}
}
if (vatNumber && address?.country === 'GB') {
vatNumber = normalisedGBVATNumber(vatNumber)
}
return {
name,
address,
companyName,
vatNumber,
collectionMethod,
billingInfoForPaymentMethod,
}
}
// =============================================================================
// ACCOUNT COMPARISON
// =============================================================================
/**
* Compare account-level fields between a Recurly account and a Stripe customer.
*
* This encapsulates the drift-detection logic shared by both the finalize script
* and the compare script.
*
* @param {object} options
* @param {object} options.account - Recurly account (npm SDK camelCase format)
* @param {object} options.stripeCustomer - Stripe customer (expanded with tax_ids, default_payment_method)
* @param {string} options.overleafUserId - Overleaf user ID (for metadata.userId comparison)
* @param {() => Promise<string|null>} options.fetchCollectionMethod - Callback to get subscription collection method
* @param {Array} options.stripePaymentMethods - Pre-fetched Stripe payment methods
* @param {string} options.stripeServiceName - 'stripe-us' or 'stripe-uk'
* @returns {Promise<Record<string, { recurly: any, stripe: any }>>} - Per-field diffs (empty = no drift)
*/
export async function compareAccountFields({
account,
stripeCustomer,
overleafUserId,
fetchCollectionMethod,
stripePaymentMethods,
stripeServiceName,
}) {
const { name, address, companyName, vatNumber } =
await resolveCustomerIdentity(account, fetchCollectionMethod)
const diffs = {}
// Email
if (
normalizeComparableString(account.email) !==
normalizeComparableString(stripeCustomer.email)
) {
diffs.email = {
recurly: account.email || null,
stripe: stripeCustomer.email || null,
}
}
// Name
if (
normalizeComparableString(name) !==
normalizeComparableString(stripeCustomer.name)
) {
diffs.name = {
recurly: name || null,
stripe: stripeCustomer.name || null,
}
}
// Address
if (address) {
if (!addressesEqual(address, stripeCustomer.address)) {
diffs.address = {
recurly: address,
stripe: stripeCustomer.address || null,
}
}
}
// Business name (company)
if (companyName) {
if (
normalizeComparableString(companyName) !==
normalizeComparableString(stripeCustomer.business_name)
) {
diffs.business_name = {
recurly: companyName,
stripe: stripeCustomer.business_name || null,
}
}
}
// Metadata
const expectedMetadata = {}
if (account.createdAt) {
expectedMetadata.recurlyCreatedAt = account.createdAt.toISOString()
}
expectedMetadata.recurlyAccountCode = ''
expectedMetadata.userId = overleafUserId
const { metadata: customFieldMetadata } =
extractRecurlyCustomFieldMetadata(account)
Object.assign(expectedMetadata, customFieldMetadata)
// Tax ID
let expectedTaxIdType = null
let expectedTaxIdValue = null
if (vatNumber) {
const taxIdTypeResult = getTaxIdType(
address?.country,
vatNumber,
address?.postal_code,
false
)
if (taxIdTypeResult.type && address?.country) {
expectedTaxIdType = taxIdTypeResult.type
expectedTaxIdValue = vatNumber
} else {
expectedMetadata.taxInfoPending = vatNumber
}
}
if (!expectedMetadata.taxInfoPending) {
expectedMetadata.taxInfoPending = ''
}
for (const [key, expectedValue] of Object.entries(expectedMetadata)) {
const stripeValue = stripeCustomer.metadata?.[key] ?? ''
if (
normalizeComparableString(expectedValue) !==
normalizeComparableString(stripeValue)
) {
diffs[`metadata.${key}`] = {
recurly: expectedValue,
stripe: stripeValue || null,
}
}
}
// Tax exempt
const expectedTaxExempt = account.taxExempt ? 'exempt' : 'none'
if (expectedTaxExempt !== (stripeCustomer.tax_exempt || 'none')) {
diffs.tax_exempt = {
recurly: expectedTaxExempt,
stripe: stripeCustomer.tax_exempt || 'none',
}
}
// CC emails
const expectedCcEmails = ccEmailsToArray(account.ccEmails)
const stripeCcEmails = stripeCustomer.additional_emails?.cc || []
const recurlyCcSorted = [...expectedCcEmails].sort()
const stripeCcSorted = [...stripeCcEmails].sort()
if (JSON.stringify(recurlyCcSorted) !== JSON.stringify(stripeCcSorted)) {
diffs.cc_emails = {
recurly: expectedCcEmails,
stripe: stripeCcEmails,
}
}
// Tax ID
if (expectedTaxIdType && expectedTaxIdValue) {
const normalizedExpectedTaxIdValue = normalizeTaxIdValue(expectedTaxIdValue)
const stripeTaxIds = stripeCustomer.tax_ids?.data || []
const matchingTaxId = stripeTaxIds.find(
tid =>
tid.type === expectedTaxIdType &&
normalizedExpectedTaxIdValue &&
normalizeTaxIdValue(tid.value) === normalizedExpectedTaxIdValue
)
if (!matchingTaxId) {
diffs.tax_id = {
recurly: { type: expectedTaxIdType, value: expectedTaxIdValue },
stripe:
stripeTaxIds.length > 0
? stripeTaxIds.map(t => ({ type: t.type, value: t.value }))
: null,
}
}
}
// Default payment method
const stripeDefaultPaymentMethod =
stripeCustomer.invoice_settings?.default_payment_method
const isPaypalBillingAgreement =
account.billingInfo?.paymentMethod?.object === 'paypal_billing_agreement'
let expectedPaymentMethod = null
if (
isPaypalBillingAgreement &&
!['US', 'CA'].includes(address?.country) &&
stripeServiceName !== 'stripe-us'
) {
expectedPaymentMethod = { type: 'paypal' }
}
if (
!isPaypalBillingAgreement &&
stripePaymentMethods.length > 0 &&
account.billingInfo &&
account.billingInfo.paymentMethod
) {
expectedPaymentMethod = account.billingInfo.paymentMethod
}
const stripePaymentMethodCreatedAt = stripeDefaultPaymentMethod?.created
const recurlyPaymentMethodUpdatedAt =
(account.billingInfo?.updatedAt?.getTime() || 0) / 1000
const recurlyPaymentMethodIsNewer =
stripePaymentMethodCreatedAt < recurlyPaymentMethodUpdatedAt
const paymentMethodIsMissing =
(!stripeDefaultPaymentMethod && expectedPaymentMethod) ||
(stripeDefaultPaymentMethod && !expectedPaymentMethod)
if (
paymentMethodIsMissing ||
(!areStripeAndRecurlyCardDetailsEqual(
stripeDefaultPaymentMethod,
expectedPaymentMethod
) &&
recurlyPaymentMethodIsNewer)
) {
diffs.default_payment_method = {
recurly: {
type: expectedPaymentMethod?.type || 'card',
last4: expectedPaymentMethod?.lastFour || null,
expiry: expectedPaymentMethod?.lastFour
? `${expectedPaymentMethod.expMonth}/${expectedPaymentMethod.expYear}`
: null,
updatedAt: account.billingInfo?.updatedAt?.toISOString() || null,
},
stripe: {
type: stripeDefaultPaymentMethod?.type || null,
last4: stripeDefaultPaymentMethod?.card
? stripeDefaultPaymentMethod.card.last4
: null,
expiry: stripeDefaultPaymentMethod?.card
? `${stripeDefaultPaymentMethod.card.exp_month}/${stripeDefaultPaymentMethod.card.exp_year}`
: null,
created: stripeDefaultPaymentMethod?.created
? new Date(stripeDefaultPaymentMethod.created * 1000).toISOString()
: null,
},
}
}
return diffs
}
@@ -37,6 +37,15 @@ import {
getTaxIdType,
coalesceOrThrowPaymentMethod,
normalisedGBVATNumber,
normalizeComparableString,
hasAnyAddressValue,
ccEmailsToArray,
normalizeTaxIdValue,
extractRecurlyCustomFieldMetadata,
addressesEqual,
resolveCustomerIdentity,
compareAccountFields,
areStripeAndRecurlyCardDetailsEqual,
} from './migrate_recurly_customers_to_stripe.helpers.mjs'
test('extractNameFromAccount returns normalized full name when first and last are present', () => {
@@ -479,3 +488,568 @@ test('normalisedGBVATNumber preserves Northern Ireland XI numbers', () => {
// XI numbers should not be modified to GB
assert.equal(normalisedGBVATNumber('XI123456789'), 'XI123456789')
})
test('normalizeComparableString returns empty string for null/undefined', () => {
assert.equal(normalizeComparableString(null), '')
assert.equal(normalizeComparableString(undefined), '')
})
test('normalizeComparableString trims whitespace', () => {
assert.equal(normalizeComparableString(' hello '), 'hello')
assert.equal(normalizeComparableString(' '), '')
})
test('normalizeComparableString converts non-strings', () => {
assert.equal(normalizeComparableString(42), '42')
assert.equal(normalizeComparableString(0), '0')
})
test('hasAnyAddressValue returns false for null/undefined/non-object', () => {
assert.equal(hasAnyAddressValue(null), false)
assert.equal(hasAnyAddressValue(undefined), false)
assert.equal(hasAnyAddressValue('string'), false)
})
test('hasAnyAddressValue returns false for all-empty address', () => {
assert.equal(hasAnyAddressValue({ line1: '', line2: '', city: '' }), false)
assert.equal(hasAnyAddressValue({ line1: ' ', city: null }), false)
})
test('hasAnyAddressValue returns true when any field is non-empty', () => {
assert.equal(hasAnyAddressValue({ line1: '123 Main St' }), true)
assert.equal(hasAnyAddressValue({ country: 'US' }), true)
})
test('ccEmailsToArray returns empty array for null/undefined', () => {
assert.deepEqual(ccEmailsToArray(null), [])
assert.deepEqual(ccEmailsToArray(undefined), [])
})
test('ccEmailsToArray splits on commas, semicolons, and whitespace', () => {
assert.deepEqual(ccEmailsToArray('a@b.com,c@d.com'), ['a@b.com', 'c@d.com'])
assert.deepEqual(ccEmailsToArray('a@b.com;c@d.com'), ['a@b.com', 'c@d.com'])
assert.deepEqual(ccEmailsToArray('a@b.com c@d.com'), ['a@b.com', 'c@d.com'])
})
test('ccEmailsToArray deduplicates', () => {
assert.deepEqual(ccEmailsToArray('a@b.com,a@b.com,c@d.com'), [
'a@b.com',
'c@d.com',
])
})
test('ccEmailsToArray handles empty string', () => {
assert.deepEqual(ccEmailsToArray(''), [])
})
test('normalizeTaxIdValue strips punctuation and lowercases', () => {
assert.equal(normalizeTaxIdValue('GB.123-456_789'), 'gb123456789')
assert.equal(normalizeTaxIdValue(' DE:123/456 '), 'de123456')
})
test('normalizeTaxIdValue handles null/undefined', () => {
assert.equal(normalizeTaxIdValue(null), '')
assert.equal(normalizeTaxIdValue(undefined), '')
})
test('extractRecurlyCustomFieldMetadata extracts known fields', () => {
const account = {
customFields: [
{ name: 'channel', value: 'web' },
{ name: 'Industry', value: 'Education' },
{ name: 'unknown_field', value: 'ignored' },
],
}
const { metadata, counts } = extractRecurlyCustomFieldMetadata(account)
assert.deepEqual(metadata, { channel: 'web', Industry: 'Education' })
assert.equal(counts.channel, 1)
assert.equal(counts.Industry, 1)
assert.equal(counts.ol_sales_person, 0)
})
test('extractRecurlyCustomFieldMetadata returns empty for no custom fields', () => {
const { metadata, counts } = extractRecurlyCustomFieldMetadata({
customFields: [],
})
assert.deepEqual(metadata, {})
assert.equal(counts.noCustomFields, 1)
})
test('extractRecurlyCustomFieldMetadata throws for missing customFields', () => {
assert.throws(
() => extractRecurlyCustomFieldMetadata({}),
/account.customFields is missing or not an array/
)
})
test('extractRecurlyCustomFieldMetadata skips null/empty values', () => {
const account = {
customFields: [
{ name: 'channel', value: null },
{ name: 'Industry', value: '' },
{ name: 'ol_sales_person', value: ' ' },
],
}
const { metadata } = extractRecurlyCustomFieldMetadata(account)
assert.deepEqual(metadata, {})
})
test('addressesEqual returns true for two null addresses', () => {
assert.equal(addressesEqual(null, null), true)
})
test('addressesEqual returns false when one is null', () => {
assert.equal(
addressesEqual({ line1: '123 Main St', country: 'US' }, null),
false
)
assert.equal(
addressesEqual(null, { line1: '123 Main St', country: 'US' }),
false
)
})
test('addressesEqual returns true for matching addresses', () => {
const a = {
line1: '123 Main St',
line2: '',
city: 'London',
state: '',
postal_code: 'SW1A 1AA',
country: 'GB',
}
const b = {
line1: '123 Main St',
line2: '',
city: 'London',
state: '',
postal_code: 'SW1A 1AA',
country: 'gb',
}
assert.equal(addressesEqual(a, b), true)
})
test('addressesEqual returns false for differing addresses', () => {
const a = {
line1: '123 Main St',
city: 'London',
country: 'GB',
}
const b = {
line1: '456 High St',
city: 'London',
country: 'GB',
}
assert.equal(addressesEqual(a, b), false)
})
test('addressesEqual normalizes whitespace', () => {
const a = { line1: ' 123 Main St ', country: ' GB ' }
const b = { line1: '123 Main St', country: 'GB' }
assert.equal(addressesEqual(a, b), true)
})
test('areStripeAndRecurlyCardDetailsEqual returns false for null inputs', () => {
assert.equal(areStripeAndRecurlyCardDetailsEqual(null, null), false)
assert.equal(
areStripeAndRecurlyCardDetailsEqual(null, { lastFour: '1234' }),
false
)
assert.equal(
areStripeAndRecurlyCardDetailsEqual(
{ type: 'card', card: { last4: '1234', exp_month: 12, exp_year: 2030 } },
null
),
false
)
})
test('areStripeAndRecurlyCardDetailsEqual returns false for non-card type', () => {
assert.equal(
areStripeAndRecurlyCardDetailsEqual(
{ type: 'paypal' },
{ lastFour: '1234' }
),
false
)
})
test('areStripeAndRecurlyCardDetailsEqual returns true for matching cards', () => {
assert.equal(
areStripeAndRecurlyCardDetailsEqual(
{ type: 'card', card: { last4: '4242', exp_month: 6, exp_year: 2025 } },
{ lastFour: '4242', expMonth: 6, expYear: 2025 }
),
true
)
})
test('areStripeAndRecurlyCardDetailsEqual returns false for mismatched last4', () => {
assert.equal(
areStripeAndRecurlyCardDetailsEqual(
{ type: 'card', card: { last4: '4242', exp_month: 6, exp_year: 2025 } },
{ lastFour: '1111', expMonth: 6, expYear: 2025 }
),
false
)
})
test('resolveCustomerIdentity uses billing info when no conflict', async () => {
const account = {
billingInfo: {
firstName: 'Bill',
lastName: 'User',
address: { street1: '123 Bill St', country: 'US' },
company: 'Bill Corp',
vatNumber: 'US123',
},
firstName: 'Account',
lastName: 'User',
address: { street1: '456 Acct St', country: 'US' },
company: 'Acct Corp',
vatNumber: 'US123',
}
// No conflict on vatNumber (same), but name differs -> conflict
const result = await resolveCustomerIdentity(account, async () => 'automatic')
assert.equal(result.name, 'Bill User')
})
test('resolveCustomerIdentity uses account info for manual collection', async () => {
const account = {
billingInfo: {
firstName: 'Bill',
lastName: 'User',
address: { street1: '123 Bill St', country: 'US' },
},
firstName: 'Account',
lastName: 'User',
address: { street1: '456 Acct St', country: 'US' },
}
const result = await resolveCustomerIdentity(account, async () => 'manual')
assert.equal(result.name, 'Account User')
assert.notEqual(result.billingInfoForPaymentMethod, null)
})
test('resolveCustomerIdentity returns billing info preferred when no conflict', async () => {
const account = {
billingInfo: {
firstName: 'Same',
lastName: 'Name',
vatNumber: 'GB123456789',
},
firstName: 'Same',
lastName: 'Name',
vatNumber: 'GB123456789',
}
const result = await resolveCustomerIdentity(account, async () => {
throw new Error('should not be called')
})
assert.equal(result.name, 'Same Name')
assert.equal(result.collectionMethod, null)
})
test('resolveCustomerIdentity does not call fetchCollectionMethod when no conflict', async () => {
let called = false
const account = {
billingInfo: { firstName: 'Only', lastName: 'Billing' },
}
await resolveCustomerIdentity(account, async () => {
called = true
return 'automatic'
})
assert.equal(called, false)
})
test('resolveCustomerIdentity throws when conflict but no collection method', async () => {
const account = {
billingInfo: {
firstName: 'Bill',
lastName: 'Name',
},
firstName: 'Acct',
lastName: 'Name',
}
await assert.rejects(
() => resolveCustomerIdentity(account, async () => null),
/no subscription found to determine collection method/
)
})
test('resolveCustomerIdentity normalises GB VAT', async () => {
const account = {
billingInfo: {
firstName: 'Test',
lastName: 'User',
address: { country: 'GB' },
vatNumber: '123456789',
},
}
const result = await resolveCustomerIdentity(account, async () => 'automatic')
assert.equal(result.vatNumber, 'GB123456789')
})
test('compareAccountFields returns empty diffs when everything matches', async () => {
const account = {
email: 'user@example.com',
firstName: 'Test',
lastName: 'User',
customFields: [],
billingInfo: {
firstName: 'Test',
lastName: 'User',
},
}
const stripeCustomer = {
email: 'user@example.com',
name: 'Test User',
metadata: {
recurlyAccountCode: '',
userId: 'user123',
taxInfoPending: '',
},
tax_exempt: 'none',
}
const diffs = await compareAccountFields({
account,
stripeCustomer,
overleafUserId: 'user123',
fetchCollectionMethod: async () => null,
stripePaymentMethods: [],
stripeServiceName: 'stripe-uk',
})
assert.deepEqual(diffs, {})
})
test('compareAccountFields detects email drift', async () => {
const account = {
email: 'new@example.com',
customFields: [],
billingInfo: {},
}
const stripeCustomer = {
email: 'old@example.com',
metadata: {
recurlyAccountCode: '',
userId: 'user123',
taxInfoPending: '',
},
tax_exempt: 'none',
}
const diffs = await compareAccountFields({
account,
stripeCustomer,
overleafUserId: 'user123',
fetchCollectionMethod: async () => null,
stripePaymentMethods: [],
stripeServiceName: 'stripe-uk',
})
assert.ok(diffs.email)
assert.equal(diffs.email.recurly, 'new@example.com')
assert.equal(diffs.email.stripe, 'old@example.com')
})
test('compareAccountFields detects name drift', async () => {
const account = {
email: 'user@example.com',
customFields: [],
billingInfo: {
firstName: 'New',
lastName: 'Name',
},
}
const stripeCustomer = {
email: 'user@example.com',
name: 'Old Name',
metadata: {
recurlyAccountCode: '',
userId: 'user123',
taxInfoPending: '',
},
tax_exempt: 'none',
}
const diffs = await compareAccountFields({
account,
stripeCustomer,
overleafUserId: 'user123',
fetchCollectionMethod: async () => null,
stripePaymentMethods: [],
stripeServiceName: 'stripe-uk',
})
assert.ok(diffs.name)
assert.equal(diffs.name.recurly, 'New Name')
assert.equal(diffs.name.stripe, 'Old Name')
})
test('compareAccountFields detects tax ID drift', async () => {
const account = {
email: 'user@example.com',
customFields: [],
billingInfo: {
firstName: 'Test',
lastName: 'User',
address: { country: 'DE' },
vatNumber: 'DE123456789',
},
}
const stripeCustomer = {
email: 'user@example.com',
name: 'Test User',
address: { country: 'DE' },
metadata: {
recurlyAccountCode: '',
userId: 'user123',
taxInfoPending: '',
},
tax_exempt: 'none',
tax_ids: { data: [] },
}
const diffs = await compareAccountFields({
account,
stripeCustomer,
overleafUserId: 'user123',
fetchCollectionMethod: async () => null,
stripePaymentMethods: [],
stripeServiceName: 'stripe-uk',
})
assert.ok(diffs.tax_id)
assert.equal(diffs.tax_id.recurly.type, 'eu_vat')
assert.equal(diffs.tax_id.recurly.value, 'DE123456789')
})
test('compareAccountFields does not report tax ID diff when values match after normalization', async () => {
const account = {
email: 'user@example.com',
customFields: [],
billingInfo: {
firstName: 'Test',
lastName: 'User',
address: { country: 'DE' },
vatNumber: 'DE-123.456.789',
},
}
const stripeCustomer = {
email: 'user@example.com',
name: 'Test User',
address: { country: 'DE' },
metadata: {
recurlyAccountCode: '',
userId: 'user123',
taxInfoPending: '',
},
tax_exempt: 'none',
tax_ids: {
data: [{ type: 'eu_vat', value: 'DE123456789' }],
},
}
const diffs = await compareAccountFields({
account,
stripeCustomer,
overleafUserId: 'user123',
fetchCollectionMethod: async () => null,
stripePaymentMethods: [],
stripeServiceName: 'stripe-uk',
})
assert.equal(diffs.tax_id, undefined)
})
test('compareAccountFields handles no payment methods on either side without crashing', async () => {
const account = {
email: 'user@example.com',
customFields: [],
billingInfo: {},
}
const stripeCustomer = {
email: 'user@example.com',
metadata: {
recurlyAccountCode: '',
userId: 'user123',
taxInfoPending: '',
},
tax_exempt: 'none',
invoice_settings: { default_payment_method: null },
}
// This should not throw (regression test for null payment method crash)
const diffs = await compareAccountFields({
account,
stripeCustomer,
overleafUserId: 'user123',
fetchCollectionMethod: async () => null,
stripePaymentMethods: [],
stripeServiceName: 'stripe-uk',
})
assert.equal(diffs.default_payment_method, undefined)
})
test('compareAccountFields detects metadata drift', async () => {
const account = {
email: 'user@example.com',
customFields: [{ name: 'channel', value: 'web' }],
billingInfo: {},
}
const stripeCustomer = {
email: 'user@example.com',
metadata: {
recurlyAccountCode: '',
userId: 'user123',
taxInfoPending: '',
channel: 'api',
},
tax_exempt: 'none',
}
const diffs = await compareAccountFields({
account,
stripeCustomer,
overleafUserId: 'user123',
fetchCollectionMethod: async () => null,
stripePaymentMethods: [],
stripeServiceName: 'stripe-uk',
})
assert.ok(diffs['metadata.channel'])
assert.equal(diffs['metadata.channel'].recurly, 'web')
assert.equal(diffs['metadata.channel'].stripe, 'api')
})
test('compareAccountFields detects tax_exempt drift', async () => {
const account = {
email: 'user@example.com',
customFields: [],
billingInfo: {},
taxExempt: true,
}
const stripeCustomer = {
email: 'user@example.com',
metadata: {
recurlyAccountCode: '',
userId: 'user123',
taxInfoPending: '',
},
tax_exempt: 'none',
}
const diffs = await compareAccountFields({
account,
stripeCustomer,
overleafUserId: 'user123',
fetchCollectionMethod: async () => null,
stripePaymentMethods: [],
stripeServiceName: 'stripe-uk',
})
assert.ok(diffs.tax_exempt)
assert.equal(diffs.tax_exempt.recurly, 'exempt')
assert.equal(diffs.tax_exempt.stripe, 'none')
})
@@ -0,0 +1,804 @@
#!/usr/bin/env node
/**
* This script compares existing Stripe customer records with data from Recurly
* to detect any drift since the last migration run.
*
* It is a read-only companion to migrate_recurly_customers_to_stripe.mjs.
* It uses the same normalization logic but makes no changes to Stripe.
*
* Input CSV format:
* recurly_account_code,target_stripe_account,stripe_customer_id
*
* Output files:
* --output (comparison file): One row per customer with match/mismatch status
* Format: recurly_account_code,target_stripe_account,stripe_customer_id,status,diffs
*
* <output>_details.json: Detailed diff for each customer with mismatches
* Format: Array of { recurly_account_code, stripe_customer_id, diffs: { field: { recurly, stripe } } }
*
* <output>_errors.csv: Records that failed
* Format: recurly_account_code,target_stripe_account,stripe_customer_id,error
*
* Usage:
* node scripts/recurly/compare_recurly_stripe_customers.mjs --input customers.csv --output comparison.csv --comparison-date 2026-03-19
*
* Options:
* --input, -i <file> Path to input CSV file (required)
* --output, -o <file> Path to output CSV file (required)
* --comparison-date <date> Date in YYYY-MM-DD to compare against (required)
* --limit, -l <n> Limit number of records processed (default: no limit)
* --concurrency, -c <n> Number of customers to process concurrently (default: 10)
* --recurly-rate-limit <n> Requests per second for Recurly (default: 10)
* --recurly-api-retries <n> Number of retries on Recurly 429s (default: 5)
* --recurly-retry-delay-ms <n> Delay between Recurly retries in ms (default: 1000)
* --stripe-rate-limit <n> Requests per second for Stripe (default: 50)
* --stripe-api-retries <n> Number of retries on Stripe 429s (default: 5)
* --stripe-retry-delay-ms <n> Delay between Stripe retries in ms (default: 1000)
* --verbose, -v Enable debug logging
*
* Note, prior to running this script, environment variables must have been loaded from config/local.env
*
* ```
* set -a
* source ../../config/local.env
* set +a
* ```
*/
import Settings from '@overleaf/settings'
import Stripe from 'stripe'
import recurly from 'recurly'
import minimist from 'minimist'
import PQueue from 'p-queue'
import fs from 'node:fs'
import * as csv from 'csv'
import { scriptRunner } from '../lib/ScriptRunner.mjs'
import { compareAccountFields } from '../helpers/migrate_recurly_customers_to_stripe.helpers.mjs'
import {
createRateLimitedApiWrappers,
DEFAULT_RECURLY_RATE_LIMIT,
DEFAULT_STRIPE_RATE_LIMIT,
DEFAULT_RECURLY_API_RETRIES,
DEFAULT_RECURLY_RETRY_DELAY_MS,
DEFAULT_STRIPE_API_RETRIES,
DEFAULT_STRIPE_RETRY_DELAY_MS,
} from '../stripe/RateLimiter.mjs'
// =============================================================================
// STRIPE CLIENT SETUP
// =============================================================================
const stripeClients = {}
function getRegionClient(region) {
const regionLower = String(region || '')
.trim()
.toLowerCase()
if (regionLower !== 'us' && regionLower !== 'uk') {
throw new Error(
`Unknown Stripe region: ${region}. Expected stripe-us or stripe-uk.`
)
}
if (stripeClients[regionLower]) return stripeClients[regionLower]
const secretKey =
regionLower === 'us'
? Settings.apis?.stripeUS?.secretKey ||
process.env.STRIPE_OL_SECRET_KEY ||
process.env.STRIPE_OL_US_SECRET_KEY
: Settings.apis?.stripeUK?.secretKey ||
process.env.STRIPE_OL_UK_SECRET_KEY
if (!secretKey || !String(secretKey).trim()) {
throw new Error(
`No Stripe secret key configured for region ${regionLower}.`
)
}
const client = new Stripe(secretKey, {
httpClient: Stripe.createFetchHttpClient(),
telemetry: false,
})
client.serviceName = `stripe-${regionLower}`
stripeClients[regionLower] = client
return client
}
// =============================================================================
// RECURLY CLIENT SETUP
// =============================================================================
const recurlyApiKey =
process.env.RECURLY_API_KEY || Settings.apis?.recurly?.apiKey
if (!recurlyApiKey) {
throw new Error(
'Recurly API key is not set. Set RECURLY_API_KEY env var or configure Settings.apis.recurly.apiKey'
)
}
const recurlyClient = new recurly.Client(recurlyApiKey)
// =============================================================================
// LOGGING UTILITIES
// =============================================================================
function timestamp() {
return new Date().toISOString()
}
function logWarn(message, context = {}) {
const contextStr =
Object.keys(context).length > 0 ? ` ${JSON.stringify(context)}` : ''
console.warn(`[${timestamp()}] WARN: ${message}${contextStr}`)
}
function logError(message, error = null, context = {}) {
const contextStr =
Object.keys(context).length > 0 ? ` ${JSON.stringify(context)}` : ''
console.error(`[${timestamp()}] ERROR: ${message}${contextStr}`)
if (error?.stack) {
console.error(`[${timestamp()}] STACK: ${error.stack}`)
}
}
let DEBUG_MODE = false
function logDebug(message, context = {}, { verboseOnly = false } = {}) {
if (verboseOnly && !DEBUG_MODE) return
const contextStr =
Object.keys(context).length > 0 ? ` ${JSON.stringify(context)}` : ''
const level = verboseOnly ? 'DEBUG' : 'INFO'
console.log(`[${timestamp()}] ${level}: ${message}${contextStr}`)
}
// =============================================================================
// DATA FETCHING
// =============================================================================
let rateLimiters
async function fetchRecurlyData(accountCode, context) {
return await rateLimiters.requestWithRetries(
'recurly',
() => recurlyClient.getAccount(`code-${accountCode}`),
context
)
}
async function fetchRecurlySubscription(accountCode, context) {
// Try live subscriptions first, then fall back to expired.
for (const state of ['live', 'expired']) {
const subscriptions = await rateLimiters.requestWithRetries(
'recurly',
async () => {
const pager = recurlyClient.listAccountSubscriptions(
`code-${accountCode}`,
{ params: { state, order: 'desc', sort: 'updated_at' } }
)
const results = []
for await (const subscription of pager.each()) {
results.push(subscription)
}
return results
},
context
)
if (subscriptions.length > 0) {
// Return the most recently updated subscription in this state
return subscriptions[0]
}
}
return null
}
async function fetchTargetStripeCustomer(
stripeClient,
stripeCustomerId,
context
) {
const customer = await rateLimiters.requestWithRetries(
stripeClient.serviceName,
() =>
stripeClient.customers.retrieve(stripeCustomerId, {
expand: ['tax_ids', 'invoice_settings.default_payment_method'],
}),
{ ...context, stripeApi: 'customers.retrieve' }
)
if (customer.deleted) {
throw new Error(`Stripe customer ${stripeCustomerId} has been deleted`)
}
return customer
}
async function fetchTargetStripeCustomerPaymentMethods(
stripeClient,
stripeCustomerId,
context
) {
const paymentMethods = await rateLimiters.requestWithRetries(
stripeClient.serviceName,
() => stripeClient.customers.listPaymentMethods(stripeCustomerId),
{ ...context, stripeApi: 'customers.listPaymentMethods' }
)
return paymentMethods.data
}
// =============================================================================
// COMPARISON LOGIC
// =============================================================================
/**
* Compare a single customer's Recurly data against Stripe.
*/
async function compareCustomer(row, rowNumber, comparisonDate) {
const {
recurly_account_code: recurlyAccountCode,
target_stripe_account: targetStripeAccount,
stripe_customer_id: stripeCustomerId,
} = row
const context = {
rowNumber,
recurlyAccountCode,
targetStripeAccount,
stripeCustomerId,
}
const stripeContext = {
rowNumber,
stripeCustomerId,
stripeAccount: targetStripeAccount,
}
const result = {
recurly_account_code: recurlyAccountCode,
target_stripe_account: targetStripeAccount,
stripe_customer_id: stripeCustomerId,
status: '', // 'match', 'mismatch', or 'error'
diffs: '',
error: '',
diffDetails: null,
}
try {
if (!recurlyAccountCode) throw new Error('Missing recurly_account_code')
if (!targetStripeAccount) throw new Error('Missing target_stripe_account')
if (!stripeCustomerId) throw new Error('Missing stripe_customer_id')
const region = String(targetStripeAccount || '')
.trim()
.toLowerCase()
.replace(/^stripe-/, '')
const stripeClient = getRegionClient(region)
const account = await fetchRecurlyData(recurlyAccountCode, context)
const recurlyAccountUpdatedAt = account.updatedAt?.getTime() || 0
// If Recurly account was not updated after the comparison date, consider it a match
if (recurlyAccountUpdatedAt <= comparisonDate.getTime()) {
result.status = 'match'
result.diffs = ''
return result
}
const stripeCustomer = await fetchTargetStripeCustomer(
stripeClient,
stripeCustomerId,
stripeContext
)
const stripePaymentMethods = await fetchTargetStripeCustomerPaymentMethods(
stripeClient,
stripeCustomerId,
stripeContext
)
const diffs = await compareAccountFields({
account,
stripeCustomer,
overleafUserId: recurlyAccountCode,
fetchCollectionMethod: async () => {
const subscription = await fetchRecurlySubscription(
recurlyAccountCode,
context
)
return subscription?.collectionMethod || null
},
stripePaymentMethods,
stripeServiceName: stripeClient.serviceName,
})
// Determine result
const diffKeys = Object.keys(diffs)
if (diffKeys.length === 0) {
result.status = 'match'
result.diffs = ''
} else {
result.status = 'mismatch'
result.diffs = diffKeys.join('; ')
result.diffDetails = {
recurly_account_code: recurlyAccountCode,
stripe_customer_id: stripeCustomerId,
target_stripe_account: targetStripeAccount,
comparison_date: comparisonDate.toISOString(),
recurly_account_updated_at: account.updatedAt?.toISOString(),
diffs,
}
logDebug('Customer has diffs', {
...context,
diffFields: diffKeys,
})
}
} catch (error) {
result.status = 'error'
const errorDetails = [error.message]
if (error.code) errorDetails.push(`code=${error.code}`)
if (error.type) errorDetails.push(`type=${error.type}`)
if (error.statusCode) errorDetails.push(`statusCode=${error.statusCode}`)
result.error = errorDetails.join('; ')
logError('Failed to compare customer', error, context)
}
return result
}
// =============================================================================
// CSV HELPERS
// =============================================================================
function formatCsvRow(columns, row) {
const values = columns.map(col => {
const raw = row[col]
const val = raw == null ? '' : String(raw)
if (val.includes(',') || val.includes('"') || val.includes('\n')) {
return `"${val.replace(/"/g, '""')}"`
}
return val
})
return values.join(',') + '\n'
}
function createJsonArrayWriter(jsonPath) {
const stream = fs.createWriteStream(jsonPath, { flags: 'w' })
stream.write('[\n')
let wroteAny = false
function write(value) {
const serialized = JSON.stringify(value, null, 2)
if (wroteAny) stream.write(',\n')
stream.write(serialized)
wroteAny = true
}
async function close() {
stream.write('\n]\n')
stream.end()
await new Promise((resolve, reject) => {
stream.on('finish', resolve)
stream.on('error', reject)
})
}
return { write, close }
}
// =============================================================================
// CLI
// =============================================================================
function usage() {
console.error(
'Compare Recurly customer data against migrated Stripe customers'
)
console.error('')
console.error('Usage:')
console.error(
' node scripts/recurly/compare_recurly_stripe_customers.mjs [options]'
)
console.error('')
console.error('Options:')
console.error(' --input, -i <file> Path to input CSV file (required)')
console.error(' --output, -o <file> Path to output CSV file (required)')
console.error(
' --comparison-date <date> Date in YYYY-MM-DD to compare against (required)'
)
console.error(
' --limit, -l <n> Limit number of records processed (default: no limit)'
)
console.error(
' --concurrency, -c <n> Number of customers to process concurrently (default: 10)'
)
console.error(
' --recurly-rate-limit <n> Requests per second for Recurly (default: 10)'
)
console.error(
' --recurly-api-retries <n> Number of retries on Recurly 429s (default: 5)'
)
console.error(
' --recurly-retry-delay-ms <n> Delay between Recurly retries in ms (default: 1000)'
)
console.error(
' --stripe-rate-limit <n> Requests per second for Stripe (default: 50)'
)
console.error(
' --stripe-api-retries <n> Number of retries on Stripe 429s (default: 5)'
)
console.error(
' --stripe-retry-delay-ms <n> Delay between Stripe retries in ms (default: 1000)'
)
console.error(' --verbose, -v Enable debug logging')
}
function parseConcurrency(value, { defaultValue = 10 } = {}) {
if (value === undefined || value === null || value === '') return defaultValue
const parsed = Number(value)
if (!Number.isFinite(parsed) || !Number.isInteger(parsed) || parsed < 1) {
throw new Error(
`Invalid --concurrency value: ${value}. Expected a positive integer.`
)
}
return parsed
}
function parseRateLimit(value, { defaultValue, name }) {
if (value === undefined || value === null || value === '') return defaultValue
const parsed = Number(value)
if (!Number.isFinite(parsed) || parsed <= 0) {
throw new Error(
`Invalid --${name} value: ${value}. Expected a positive number.`
)
}
return parsed
}
function parseNonNegativeInt(value, { defaultValue, name }) {
if (value === undefined || value === null || value === '') return defaultValue
const parsed = Number(value)
if (!Number.isFinite(parsed) || !Number.isInteger(parsed) || parsed < 0) {
throw new Error(
`Invalid --${name} value: ${value}. Expected a non-negative integer.`
)
}
return parsed
}
function parseArgs() {
return minimist(process.argv.slice(2), {
alias: {
i: 'input',
o: 'output',
h: 'help',
v: 'verbose',
c: 'concurrency',
l: 'limit',
},
string: [
'input',
'output',
'comparison-date',
'limit',
'recurly-rate-limit',
'recurly-api-retries',
'recurly-retry-delay-ms',
'stripe-rate-limit',
'stripe-api-retries',
'stripe-retry-delay-ms',
],
boolean: ['verbose', 'help'],
default: {
verbose: false,
concurrency: 10,
'recurly-rate-limit': DEFAULT_RECURLY_RATE_LIMIT,
'recurly-api-retries': DEFAULT_RECURLY_API_RETRIES,
'recurly-retry-delay-ms': DEFAULT_RECURLY_RETRY_DELAY_MS,
'stripe-rate-limit': DEFAULT_STRIPE_RATE_LIMIT,
'stripe-api-retries': DEFAULT_STRIPE_API_RETRIES,
'stripe-retry-delay-ms': DEFAULT_STRIPE_RETRY_DELAY_MS,
},
})
}
// =============================================================================
// MAIN
// =============================================================================
async function main(trackProgress) {
const startTime = new Date()
const args = parseArgs()
const {
input: inputPath,
output: outputPath,
'comparison-date': comparisonDateRaw,
verbose,
help,
concurrency: concurrencyRaw,
limit: limitRaw,
'recurly-rate-limit': recurlyRateLimitRaw,
'recurly-api-retries': recurlyApiRetriesRaw,
'recurly-retry-delay-ms': recurlyRetryDelayMsRaw,
'stripe-rate-limit': stripeRateLimitRaw,
'stripe-api-retries': stripeApiRetriesRaw,
'stripe-retry-delay-ms': stripeRetryDelayMsRaw,
} = args
let concurrency,
recurlyRateLimit,
recurlyApiRetriesValue,
recurlyRetryDelayMsValue,
stripeRateLimitPerSecond,
stripeApiRetriesValue,
stripeRetryDelayMsValue,
limit,
comparisonDate
try {
concurrency = parseConcurrency(concurrencyRaw, { defaultValue: 10 })
limit = parseNonNegativeInt(limitRaw, {
defaultValue: null,
name: 'limit',
})
recurlyRateLimit = parseRateLimit(recurlyRateLimitRaw, {
defaultValue: DEFAULT_RECURLY_RATE_LIMIT,
name: 'recurly-rate-limit',
})
recurlyApiRetriesValue = parseNonNegativeInt(recurlyApiRetriesRaw, {
defaultValue: DEFAULT_RECURLY_API_RETRIES,
name: 'recurly-api-retries',
})
recurlyRetryDelayMsValue = parseNonNegativeInt(recurlyRetryDelayMsRaw, {
defaultValue: DEFAULT_RECURLY_RETRY_DELAY_MS,
name: 'recurly-retry-delay-ms',
})
stripeRateLimitPerSecond = parseRateLimit(stripeRateLimitRaw, {
defaultValue: DEFAULT_STRIPE_RATE_LIMIT,
name: 'stripe-rate-limit',
})
stripeApiRetriesValue = parseNonNegativeInt(stripeApiRetriesRaw, {
defaultValue: DEFAULT_STRIPE_API_RETRIES,
name: 'stripe-api-retries',
})
stripeRetryDelayMsValue = parseNonNegativeInt(stripeRetryDelayMsRaw, {
defaultValue: DEFAULT_STRIPE_RETRY_DELAY_MS,
name: 'stripe-retry-delay-ms',
})
if (!comparisonDateRaw) {
throw new Error('--comparison-date is required')
}
const dateMatch = comparisonDateRaw.match(/^(\d{4})-(\d{2})-(\d{2})$/)
if (!dateMatch) {
throw new Error('--comparison-date must be in YYYY-MM-DD format')
}
const [, year, month, day] = dateMatch
comparisonDate = new Date(`${year}-${month}-${day}T00:00:00.000Z`)
if (isNaN(comparisonDate.getTime())) {
throw new Error('Invalid date provided for --comparison-date')
}
} catch (error) {
logError(error.message)
usage()
process.exit(1)
}
rateLimiters = createRateLimitedApiWrappers({
recurlyRateLimit,
recurlyApiRetries: recurlyApiRetriesValue,
recurlyRetryDelayMs: recurlyRetryDelayMsValue,
stripeRateLimit: stripeRateLimitPerSecond,
stripeApiRetries: stripeApiRetriesValue,
stripeRetryDelayMs: stripeRetryDelayMsValue,
logDebug,
logWarn,
})
DEBUG_MODE = !!verbose
if (help || !inputPath || !outputPath) {
usage()
process.exit(help ? 0 : 1)
}
const errorsPath = outputPath.replace(/\.csv$/, '_errors.csv')
const detailsJsonPath = outputPath.replace(/\.csv$/, '_details.json')
logDebug('Starting comparison', {
inputPath,
outputPath,
errorsPath,
detailsJsonPath,
comparisonDate: comparisonDate.toISOString(),
concurrency,
...(limit != null ? { limit } : {}),
})
await trackProgress('Starting comparison')
// Output CSV columns
const outputColumns = [
'recurly_account_code',
'target_stripe_account',
'stripe_customer_id',
'status',
'diffs',
]
const errorColumns = [
'recurly_account_code',
'target_stripe_account',
'stripe_customer_id',
'error',
]
const outputStream = fs.createWriteStream(outputPath, { flags: 'w' })
outputStream.write(outputColumns.join(',') + '\n')
const errorsStream = fs.createWriteStream(errorsPath, { flags: 'w' })
errorsStream.write(errorColumns.join(',') + '\n')
const detailsWriter = createJsonArrayWriter(detailsJsonPath)
try {
let totalInInput = 0
let processedCount = 0
let matchCount = 0
let mismatchCount = 0
let errorCount = 0
let queuedCount = 0
const inputStream = fs.createReadStream(inputPath)
const parser = csv.parse({
columns: true,
trim: true,
bom: true,
skip_empty_lines: true,
relax_column_count: true,
relax_column_count_less: true,
})
inputStream.pipe(parser)
const queue = new PQueue({ concurrency })
const maxQueueSize = concurrency
let rowNumber = 0
let limitReached = false
try {
for await (const row of parser) {
rowNumber++
totalInInput++
const thisRowNumber = rowNumber
if (limit != null && queuedCount >= limit) {
limitReached = true
logDebug('Record limit reached', { limit, queuedCount })
break
}
if (queue.size >= maxQueueSize) {
await queue.onSizeLessThan(maxQueueSize)
}
queuedCount++
queue.add(async () => {
let result
try {
result = await compareCustomer(row, thisRowNumber, comparisonDate)
} catch (error) {
result = {
...row,
status: 'error',
diffs: '',
error: error?.message || String(error),
diffDetails: null,
}
logError('Unhandled error', error, {
rowNumber: thisRowNumber,
accountCode: row.recurly_account_code,
})
}
processedCount++
if (result.status === 'match') {
matchCount++
outputStream.write(formatCsvRow(outputColumns, result))
} else if (result.status === 'mismatch') {
mismatchCount++
outputStream.write(formatCsvRow(outputColumns, result))
if (result.diffDetails) {
detailsWriter.write(result.diffDetails)
}
} else {
errorCount++
errorsStream.write(formatCsvRow(errorColumns, result))
}
const progressInterval = DEBUG_MODE ? 100 : 1000
if (processedCount % progressInterval === 0) {
logDebug('Progress', {
processedCount,
matchCount,
mismatchCount,
errorCount,
})
await trackProgress(
`Progress: ${processedCount} processed, ${matchCount} match, ${mismatchCount} mismatch, ${errorCount} errors`
)
}
})
}
} finally {
await queue.onIdle()
}
if (limitReached) {
await trackProgress(`Limit reached (${limit}).`)
}
// Final summary
const endTime = new Date()
const durationMs = endTime.getTime() - startTime.getTime()
const durationSeconds = Math.floor(durationMs / 1000)
const finalStats = rateLimiters.getRateLimiterStats()
await trackProgress('=== COMPARISON SUMMARY ===')
await trackProgress(`Total in input: ${totalInInput}`)
await trackProgress(`Processed: ${processedCount}`)
await trackProgress(` Match: ${matchCount}`)
await trackProgress(` Mismatch: ${mismatchCount}`)
await trackProgress(` Error: ${errorCount}`)
await trackProgress(`Duration: ${durationSeconds}s`)
await trackProgress(`Output: ${outputPath}`)
await trackProgress(`Errors: ${errorsPath} (${errorCount} records)`)
await trackProgress(
`Details: ${detailsJsonPath} (${mismatchCount} records)`
)
await trackProgress(
`API calls - Recurly: ${finalStats.recurly.totalRequests}, Stripe: ${finalStats.stripe.totalRequests}`
)
logDebug('Comparison complete', {
totalInInput,
processedCount,
matchCount,
mismatchCount,
errorCount,
})
return errorCount === 0 && mismatchCount === 0 ? 0 : 1
} finally {
outputStream.end()
errorsStream.end()
const results = await Promise.allSettled([
new Promise((resolve, reject) => {
outputStream.on('finish', resolve)
outputStream.on('error', reject)
}),
new Promise((resolve, reject) => {
errorsStream.on('finish', resolve)
errorsStream.on('error', reject)
}),
detailsWriter.close(),
])
for (const result of results) {
if (result.status === 'rejected') {
logWarn('Failed to close output stream', {
error: result.reason?.message || String(result.reason),
})
}
}
}
}
try {
const exitCode = await scriptRunner(main)
process.exit(exitCode ?? 0)
} catch (error) {
logError('Script failed with unhandled error', error)
process.exit(1)
}
@@ -93,13 +93,17 @@ import { scriptRunner } from '../lib/ScriptRunner.mjs'
import {
areStripeAndRecurlyCardDetailsEqual,
coalesceOrThrowPaymentMethod,
extractNameFromAccount,
extractNameFromBillingInfo,
getTaxIdType,
normalisedGBVATNumber,
normalizeRecurlyAddressToStripe,
normalizeName,
addressesEqual,
resolveCustomerIdentity,
sanitizeAccount,
normalizeComparableString,
hasAnyAddressValue,
ccEmailsToArray,
RECURLY_CUSTOM_FIELD_NAMES,
extractRecurlyCustomFieldMetadata,
} from '../helpers/migrate_recurly_customers_to_stripe.helpers.mjs'
import {
createRateLimitedApiWrappers,
@@ -182,66 +186,6 @@ if (!recurlyApiKey) {
}
const recurlyClient = new recurly.Client(recurlyApiKey)
// =============================================================================
// CUSTOM FIELDS
// =============================================================================
const RECURLY_CUSTOM_FIELD_NAMES = [
'channel',
'Industry',
'ol_sales_person',
'MigratedfromFreeAgent',
]
function getRecurlyCustomFields(account) {
const customFields = account?.customFields
if (customFields == null) {
throw new Error(
'Recurly account is missing customFields (empty array is acceptable)'
)
}
if (!Array.isArray(customFields)) {
throw new Error('Recurly account customFields is not an array')
}
return customFields
}
function extractRecurlyCustomFieldMetadata(account) {
const customFields = getRecurlyCustomFields(account)
/** @type {Record<string, string>} */
const metadata = {}
const counts = {
channel: 0,
Industry: 0,
ol_sales_person: 0,
MigratedfromFreeAgent: 0,
noCustomFields: 0,
}
if (customFields.length === 0) {
counts.noCustomFields = 1
return { metadata, counts }
}
for (const field of customFields) {
const name = field?.name?.trim()
if (!RECURLY_CUSTOM_FIELD_NAMES.includes(name)) continue
const rawValue = field?.value
if (rawValue == null) continue
const value = String(rawValue).trim()
if (!value) continue
metadata[name] = value
counts[name] = 1
}
return { metadata, counts }
}
// =============================================================================
// LOGGING UTILITIES
// =============================================================================
@@ -926,65 +870,8 @@ function isStripeTaxIdInvalidError(error) {
return error.code === 'tax_id_invalid'
}
function normalizeComparableString(value) {
if (value == null) return ''
return String(value).trim()
}
const STRIPE_METADATA_MAX_ALT_EMAILS = 5
function ccEmailsToArray(ccEmails) {
if (ccEmails == null || ccEmails === undefined) {
return []
}
// regex splits on commas, semicolons or whitespace and trims each email
// empty values are filtered out
const normalisedEmails = String(ccEmails)
.split(/[\s,;]+/)
.filter(Boolean)
const deDupedEmails = [...new Set(normalisedEmails)]
return deDupedEmails
}
function hasAnyAddressValue(address) {
if (!address || typeof address !== 'object') return false
return Object.values(address).some(v => normalizeComparableString(v) !== '')
}
function normalizeComparableAddress(address) {
if (!address || typeof address !== 'object') return null
const norm = {
line1: normalizeComparableString(address.line1),
line2: normalizeComparableString(address.line2),
city: normalizeComparableString(address.city),
state: normalizeComparableString(address.state),
postal_code: normalizeComparableString(address.postal_code),
country: normalizeComparableString(address.country).toUpperCase(),
}
return hasAnyAddressValue(norm) ? norm : null
}
function addressesEqual(a, b) {
const na = normalizeComparableAddress(a)
const nb = normalizeComparableAddress(b)
if (!na && !nb) return true
if (!na || !nb) return false
return (
na.line1 === nb.line1 &&
na.line2 === nb.line2 &&
na.city === nb.city &&
na.state === nb.state &&
na.postal_code === nb.postal_code &&
na.country === nb.country
)
}
// =============================================================================
// MAIN PROCESSING
// =============================================================================
@@ -1216,153 +1103,6 @@ async function updatePaymentMethodBillingDetails(
)
}
/**
* Resolve customer name, address, company, and VAT number from Recurly account data.
*
* For most customers, billing info and account info agree (or only one source is set),
* and the standard coalesce logic applies (billing info preferred, account as fallback).
*
* When any field has conflicting values across both sources, the subscription's
* collection_method is fetched to determine which source wins:
*
* - automatic (web sales): billing info is used for the Stripe customer record.
* - manual (manual billing): account info is used for the Stripe customer record,
* and billing info is returned separately to be copied to the payment method's
* billing_details.
*
* @param {object} account - Recurly account object
* @param {string} recurlyAccountCode - Account code (used for subscription lookup on conflict)
* @param {object} context - Logging context
* @returns {Promise<{
* name: string|null,
* address: import('stripe').Stripe.AddressParam|null,
* companyName: string|null,
* vatNumber: string|null,
* collectionMethod: string|null,
* billingInfoForPaymentMethod: object|null
* }>}
*/
async function resolveCustomerIdentity(account, recurlyAccountCode, context) {
// Detect conflicts between billing info and account fields
const billingName = extractNameFromBillingInfo(account)
const accountName = extractNameFromAccount(account)
const nameConflict =
billingName !== null && accountName !== null && billingName !== accountName
const billingAddress = normalizeRecurlyAddressToStripe(
account.billingInfo?.address
)
const accountAddress = normalizeRecurlyAddressToStripe(account?.address)
const addressConflict =
billingAddress !== null &&
accountAddress !== null &&
!addressesEqual(billingAddress, accountAddress)
const billingCompany = account.billingInfo?.company?.trim() || null
const accountCompany = account.company?.trim() || null
const companyConflict =
billingCompany !== null &&
accountCompany !== null &&
billingCompany !== accountCompany
const billingVat = account.billingInfo?.vatNumber?.trim() || null
const accountVat = account?.vatNumber?.trim() || null
const vatConflict =
billingVat !== null && accountVat !== null && billingVat !== accountVat
const hasConflict =
nameConflict || addressConflict || companyConflict || vatConflict
let name,
address,
companyName,
vatNumber,
collectionMethod,
billingInfoForPaymentMethod
if (!hasConflict) {
// No conflict: use the standard coalesce logic (billing info preferred)
name = billingName ?? accountName
address = billingAddress ?? accountAddress
companyName = billingCompany ?? accountCompany
vatNumber = billingVat ?? accountVat
collectionMethod = null
billingInfoForPaymentMethod = null
} else {
// Conflict detected: fetch the subscription to determine which source wins
logWarn(
'Conflict between billing info and account fields; fetching subscription collection method to resolve',
{
...context,
nameConflict,
addressConflict,
companyConflict,
vatConflict,
}
)
const subscription = await fetchRecurlyActiveSubscription(
recurlyAccountCode,
context
)
collectionMethod = subscription?.collectionMethod || null
if (!collectionMethod) {
throw new Error(
'Conflict between billing info and account fields, but no live subscription found to determine collection method'
)
}
logDebug(
'Resolving billing info / account conflict using subscription collection method',
{
...context,
collectionMethod,
nameConflict,
addressConflict,
companyConflict,
vatConflict,
},
{ verboseOnly: true }
)
if (collectionMethod === 'automatic') {
// Web sales: use billing info for the Stripe customer record
name = billingName ?? accountName
address = billingAddress ?? accountAddress
companyName = billingCompany ?? accountCompany
vatNumber = billingVat ?? accountVat
billingInfoForPaymentMethod = null
} else if (collectionMethod === 'manual') {
// Manual billing: use account info for the Stripe customer record,
// and return billing info to be copied to the payment method's billing_details
name = accountName ?? billingName
address = accountAddress ?? billingAddress
companyName = accountCompany ?? billingCompany
vatNumber = accountVat ?? billingVat
billingInfoForPaymentMethod = account.billingInfo
} else {
throw new Error(
`Unknown collection method "${collectionMethod}" encountered while resolving billing info / account conflict`
)
}
}
// Normalise GB VAT numbers using the resolved address country
if (vatNumber && address?.country === 'GB') {
vatNumber = normalisedGBVATNumber(vatNumber)
}
return {
name,
address,
companyName,
vatNumber,
collectionMethod,
billingInfoForPaymentMethod,
}
}
/**
* Process a single customer row from the input CSV.
*
@@ -1485,7 +1225,23 @@ async function processCustomer(
companyName,
vatNumber,
billingInfoForPaymentMethod,
} = await resolveCustomerIdentity(account, recurlyAccountCode, context)
} = await resolveCustomerIdentity(account, async () => {
logWarn(
'Conflict between billing info and account fields; fetching subscription collection method to resolve',
{ ...context }
)
const subscription = await fetchRecurlyActiveSubscription(
recurlyAccountCode,
context
)
const cm = subscription?.collectionMethod || null
logDebug(
'Resolved collection method for conflict',
{ ...context, collectionMethod: cm },
{ verboseOnly: true }
)
return cm
})
if (name === null && companyName === null) {
// This should not happen since we're handling all the known cases in resolveCustomerIdentity but just in case
@@ -39,6 +39,8 @@ import fs from 'node:fs'
import path from 'node:path'
import * as csv from 'csv'
import minimist from 'minimist'
import Settings from '@overleaf/settings'
import recurly from 'recurly'
import PQueue from 'p-queue'
import { z } from '../../app/src/infrastructure/Validation.mjs'
import { scriptRunner } from '../lib/ScriptRunner.mjs'
@@ -56,6 +58,7 @@ import UserAnalyticsIdCache from '../../app/src/Features/Analytics/UserAnalytics
import CustomerIoHandler from '../../modules/customer-io/app/src/CustomerIoHandler.mjs'
import { ReportError } from './helpers.mjs'
import isEqual from 'lodash/isEqual.js'
import { compareAccountFields } from '../helpers/migrate_recurly_customers_to_stripe.helpers.mjs'
import {
createRateLimitedApiWrappers,
DEFAULT_RECURLY_RATE_LIMIT,
@@ -71,6 +74,16 @@ const preloadedProductMetadata = new Map()
// rate limiters - initialized in main()
let rateLimiters
// Recurly SDK client - initialized at module level
const recurlyApiKey =
process.env.RECURLY_API_KEY || Settings.apis?.recurly?.apiKey
if (!recurlyApiKey) {
throw new Error(
'Recurly API key is not set. Set RECURLY_API_KEY env var or configure Settings.apis.recurly.apiKey'
)
}
const recurlyClient = new recurly.Client(recurlyApiKey)
function usage() {
console.error(`Usage: node scripts/stripe/finalize-stripe-subscription-migration.mjs [OPTS] [INPUT-FILE]
@@ -366,16 +379,13 @@ async function processMigration(input, commit) {
)
}
// 5. Fetch Recurly subscription
// 5. Fetch Recurly subscription and account
let recurlySubscription
try {
recurlySubscription = await rateLimiters.requestWithRetries(
'recurly',
() =>
RecurlyWrapper.promises.getSubscription(
previousRecurlySubscriptionId,
{}
),
recurlyClient.getSubscription(`uuid-${previousRecurlySubscriptionId}`),
{
operation: 'getSubscription',
recurlySubscriptionId: previousRecurlySubscriptionId,
@@ -388,12 +398,41 @@ async function processMigration(input, commit) {
)
}
let recurlyAccount
try {
recurlyAccount = await rateLimiters.requestWithRetries(
'recurly',
() => recurlyClient.getAccount(`code-${overleafUserId}`),
{
operation: 'getAccount',
overleafUserId,
}
)
} catch (err) {
throw new ReportError(
'no-recurly-account',
`Recurly account not found: ${err.message}`
)
}
// 6. Detect changes between Recurly and Stripe
const changes = detectChanges(recurlySubscription, stripeSubscription, region)
if (changes.length > 0) {
const subscriptionChanges = detectSubscriptionChanges(
recurlySubscription,
stripeSubscription,
region
)
const accountChanges = await detectAccountChanges(
overleafUserId,
stripeCustomerId,
stripeClient,
recurlyAccount,
recurlySubscription.collectionMethod || null
)
const allChanges = [...subscriptionChanges, ...accountChanges]
if (allChanges.length > 0) {
throw new ReportError(
'changes-detected',
`Changes detected between Recurly and Stripe: ${changes.join('; ')}`
`Changes detected between Recurly and Stripe: ${allChanges.join('; ')}`
)
}
@@ -450,38 +489,42 @@ function formatItems(items) {
.join(', ')
}
function detectChanges(recurlySubscription, stripeSubscription, region) {
function detectSubscriptionChanges(
recurlySubscription,
stripeSubscription,
region
) {
const changes = []
// Extract item details from Recurly subscription
const targetRecurlySubscription =
recurlySubscription.pending_subscription || recurlySubscription
recurlySubscription.pendingChange || recurlySubscription
const recurlyPlanItem =
PlansLocator.convertLegacyGroupPlanCodeToConsolidatedGroupPlanCodeIfNeeded(
targetRecurlySubscription.plan.plan_code
targetRecurlySubscription.plan.code
)
const simplifiedPlanCode = recurlyPlanItem.planCode.replace(
/_free_trial.*$/,
''
)
const additionalLicenseQuantity =
(targetRecurlySubscription.subscription_add_ons || []).find(
addOn => addOn.add_on_code === 'additional-license'
(targetRecurlySubscription.addOns || []).find(
addOn => addOn.addOn.code === 'additional-license'
)?.quantity || 0
const recurlyItems = [
{
code: simplifiedPlanCode,
quantity: recurlyPlanItem.quantity + additionalLicenseQuantity,
amount:
targetRecurlySubscription.unit_amount_in_cents /
Math.round(targetRecurlySubscription.unitAmount * 100) /
recurlyPlanItem.quantity,
},
...(targetRecurlySubscription.subscription_add_ons || [])
.filter(addOn => addOn.add_on_code !== 'additional-license')
...(targetRecurlySubscription.addOns || [])
.filter(addOn => addOn.addOn.code !== 'additional-license')
.map(addOn => ({
code: addOn.add_on_code,
code: addOn.addOn.code,
quantity: addOn.quantity,
amount: addOn.unit_amount_in_cents,
amount: Math.round(addOn.unitAmount * 100),
})),
].sort((a, b) => a.code.localeCompare(b.code))
@@ -526,6 +569,109 @@ function detectChanges(recurlySubscription, stripeSubscription, region) {
return changes
}
/**
* Detect account-level drift between the Recurly account and the migrated Stripe customer.
*
* Uses the Recurly SDK account (which includes billing info), and re-retrieves
* the Stripe customer with expanded tax_ids and default_payment_method so the
* comparison can cover all the fields that the customer-migration script set.
*
* @param {string} overleafUserId - Recurly account code / Overleaf user ID
* @param {string} stripeCustomerId - Stripe customer ID
* @param {object} stripeClient - Stripe client (from getRegionClient)
* @param {object} account - Recurly SDK account object (from recurlyClient.getAccount)
* @param {string|null} collectionMethod - Recurly subscription collection method
* @returns {Promise<string[]>} - Array of change descriptions (empty = no drift)
*/
async function detectAccountChanges(
overleafUserId,
stripeCustomerId,
stripeClient,
account,
collectionMethod
) {
const context = { overleafUserId, stripeCustomerId }
// Fetch the Stripe customer with tax_ids and payment method expanded
const stripeCustomer = await rateLimiters.requestWithRetries(
stripeClient.serviceName,
() =>
stripeClient.stripe.customers.retrieve(stripeCustomerId, {
expand: ['tax_ids', 'invoice_settings.default_payment_method'],
}),
{ ...context, operation: 'customers.retrieve' }
)
if (stripeCustomer.deleted) {
return [`Stripe customer ${stripeCustomerId} has been deleted`]
}
// Pre-fetch payment methods if needed for comparison
let stripePaymentMethods = []
const isPaypalBillingAgreement =
account.billingInfo?.paymentMethod?.object === 'paypal_billing_agreement'
if (!isPaypalBillingAgreement && account.billingInfo?.paymentMethod) {
const result = await rateLimiters.requestWithRetries(
stripeClient.serviceName,
() => stripeClient.stripe.customers.listPaymentMethods(stripeCustomerId),
{ ...context, operation: 'customers.listPaymentMethods' }
)
stripePaymentMethods = result.data
}
const diffs = await compareAccountFields({
account,
stripeCustomer,
overleafUserId,
fetchCollectionMethod: async () => collectionMethod,
stripePaymentMethods,
stripeServiceName: stripeClient.serviceName,
})
return formatDiffsAsChanges(diffs)
}
/**
* Convert structured diffs from compareAccountFields into human-readable change descriptions.
*/
function formatDiffsAsChanges(diffs) {
const changes = []
for (const [field, diff] of Object.entries(diffs)) {
if (field === 'address') {
changes.push(
`Address: Recurly=${JSON.stringify(diff.recurly)}, Stripe=${JSON.stringify(diff.stripe)}`
)
} else if (field === 'cc_emails') {
changes.push(
`CC emails: Recurly=[${[...diff.recurly].sort().join(',')}], Stripe=[${[...(diff.stripe || [])].sort().join(',')}]`
)
} else if (field === 'tax_id') {
const stripeStr = diff.stripe
? diff.stripe.map(t => `{type:${t.type}, value:${t.value}}`).join(', ')
: '(none)'
changes.push(
`Tax ID: Recurly={type:${diff.recurly.type}, value:${diff.recurly.value}}, Stripe=${stripeStr}`
)
} else if (field === 'default_payment_method') {
changes.push(
`Payment method: Recurly=${diff.recurly.type || diff.recurly.last4 || '(none)'}, Stripe=${diff.stripe.type || '(none)'}`
)
} else if (field.startsWith('metadata.')) {
const key = field.slice('metadata.'.length)
changes.push(
`Metadata ${key}: Recurly=${diff.recurly || '(empty)'}, Stripe=${diff.stripe || '(empty)'}`
)
} else {
const label =
field.charAt(0).toUpperCase() + field.slice(1).replace(/_/g, ' ')
changes.push(
`${label}: Recurly=${diff.recurly || '(empty)'}, Stripe=${diff.stripe || '(empty)'}`
)
}
}
return changes
}
async function performCutover(
mongoSubscription,
stripeSubscription,
@@ -568,9 +714,7 @@ async function performCutover(
// Step 3: Postpone Recurly billing by +10 years if Recurly subscription is active
if (recurlySubscription.state !== 'canceled') {
const currentBillingDate = new Date(
recurlySubscription.current_period_ends_at
)
const currentBillingDate = new Date(recurlySubscription.currentPeriodEndsAt)
const postponedDate = new Date(currentBillingDate)
postponedDate.setFullYear(currentBillingDate.getFullYear() + 10)