From 4d338efe561909acc818353f7d4f52ad8303a278 Mon Sep 17 00:00:00 2001 From: Simon Gardner Date: Thu, 5 Feb 2026 12:40:00 +0000 Subject: [PATCH] updates to recurly -> stripe customer upsert script GitOrigin-RevId: dfe277807f9282804fed5a9cdf86e654a719de96 --- ...te_recurly_customers_to_stripe.helpers.mjs | 657 +++++++++++-- ..._customers_to_stripe.helpers.node.test.mjs | 122 +++ .../migrate_recurly_customers_to_stripe.mjs | 865 +++++++++++++++--- 3 files changed, 1432 insertions(+), 212 deletions(-) diff --git a/services/web/scripts/helpers/migrate_recurly_customers_to_stripe.helpers.mjs b/services/web/scripts/helpers/migrate_recurly_customers_to_stripe.helpers.mjs index 8e14e24073..49146b9eef 100644 --- a/services/web/scripts/helpers/migrate_recurly_customers_to_stripe.helpers.mjs +++ b/services/web/scripts/helpers/migrate_recurly_customers_to_stripe.helpers.mjs @@ -2,7 +2,6 @@ import lodash from 'lodash' /* - * This helper can be used by migrate_recurly_customers_to_stripe.mjs * * This file can be deleted once the Recurly to Stripe migration is complete. */ @@ -181,6 +180,30 @@ export const EU_VAT_COUNTRIES = [ 'SK', // Slovakia ] +const EU_VAT_PREFIX_OVERRIDES = { + GR: 'EL', // Greece uses EL prefix for VAT numbers +} + +function normalizeTaxId(value) { + if (!value) return '' + return String(value).trim().toUpperCase() +} + +function normalizeTaxIdCompact(value) { + return normalizeTaxId(value).replace(/\s+/g, '') +} + +function digitsOnly(value) { + return normalizeTaxId(value).replace(/\D+/g, '') +} + +function hasEuVatPrefix(country, taxIdValue) { + const normalized = normalizeTaxIdCompact(taxIdValue) + if (!normalized) return false + const prefix = EU_VAT_PREFIX_OVERRIDES[country] || country + return normalized.startsWith(prefix) +} + function caProvinceFromPostalCode(postalCode) { if (!postalCode) return null const m = String(postalCode) @@ -267,6 +290,445 @@ export function getCanadaTaxIdType(taxIdValue, postalCode) { return null } +/** + * Determine the Stripe tax ID type for Australia based on the tax ID value. + * + * Source reference: + * - Stripe docs list supported types + example formats (Australia section) + * https://docs.stripe.com/billing/customer/tax-ids + * + * @param {string|undefined|null} taxIdValue + * @returns {'au_abn'|'au_arn'|null} + */ +export function getAustraliaTaxIdType(taxIdValue) { + const digits = digitsOnly(taxIdValue) + if (!digits) return null + + // ABN: 11 digits (example: 12345678912) + if (/^\d{11}$/.test(digits)) return 'au_abn' + + // ARN: 12 digits (example: 123456789123) + if (/^\d{12}$/.test(digits)) return 'au_arn' + + return null +} + +/** + * Determine the Stripe tax ID type for Brazil based on the tax ID value. + * + * Source reference: + * - Stripe docs list supported types + example formats (Brazil section) + * https://docs.stripe.com/billing/customer/tax-ids + * + * @param {string|undefined|null} taxIdValue + * @returns {'br_cnpj'|'br_cpf'|null} + */ +export function getBrazilTaxIdType(taxIdValue) { + const digits = digitsOnly(taxIdValue) + if (!digits) return null + + // CNPJ: 14 digits (example: 01.234.456/5432-10) + if (/^\d{14}$/.test(digits)) return 'br_cnpj' + + // CPF: 11 digits (example: 123.456.789-87) + if (/^\d{11}$/.test(digits)) return 'br_cpf' + + return null +} + +/** + * Determine the Stripe tax ID type for Bulgaria (EU VAT vs UIC). + * + * Source reference: + * - Stripe docs list supported types + example formats (Bulgaria section) + * https://docs.stripe.com/billing/customer/tax-ids + * + * @param {string|undefined|null} taxIdValue + * @returns {'eu_vat'|'bg_uic'|null} + */ +export function getBulgariaTaxIdType(taxIdValue) { + if (!taxIdValue) return null + + // EU VAT: BG prefix (example: BG0123456789) + if (hasEuVatPrefix('BG', taxIdValue)) return 'eu_vat' + + // UIC: 9 digits (example: 123456789) + if (/^\d{9}$/.test(digitsOnly(taxIdValue))) return 'bg_uic' + + return null +} + +/** + * Determine the Stripe tax ID type for Croatia (EU VAT vs OIB). + * + * Source reference: + * - Stripe docs list supported types + example formats (Croatia section) + * https://docs.stripe.com/billing/customer/tax-ids + * + * @param {string|undefined|null} taxIdValue + * @returns {'eu_vat'|'hr_oib'|null} + */ +export function getCroatiaTaxIdType(taxIdValue) { + if (!taxIdValue) return null + + // EU VAT: HR prefix (example: HR12345678912) + if (hasEuVatPrefix('HR', taxIdValue)) return 'eu_vat' + + // OIB: 11 digits (example: 12345678901) + if (/^\d{11}$/.test(digitsOnly(taxIdValue))) return 'hr_oib' + + return null +} + +/** + * Determine the Stripe tax ID type for Germany (EU VAT vs Steuer Nummer). + * + * Source reference: + * - Stripe docs list supported types + example formats (Germany section) + * https://docs.stripe.com/billing/customer/tax-ids + * + * @param {string|undefined|null} taxIdValue + * @returns {'eu_vat'|'de_stn'|null} + */ +export function getGermanyTaxIdType(taxIdValue) { + if (!taxIdValue) return null + + // EU VAT: DE prefix (example: DE123456789) + if (hasEuVatPrefix('DE', taxIdValue)) return 'eu_vat' + + // Steuer Nummer: 10 digits (example: 1234567890) + if (/^\d{10}$/.test(digitsOnly(taxIdValue))) return 'de_stn' + + return null +} + +/** + * Determine the Stripe tax ID type for Hungary (EU VAT vs HU tax number). + * + * Source reference: + * - Stripe docs list supported types + example formats (Hungary section) + * https://docs.stripe.com/billing/customer/tax-ids + * + * @param {string|undefined|null} taxIdValue + * @returns {'eu_vat'|'hu_tin'|null} + */ +export function getHungaryTaxIdType(taxIdValue) { + if (!taxIdValue) return null + + // EU VAT: HU prefix (example: HU12345678) + if (hasEuVatPrefix('HU', taxIdValue)) return 'eu_vat' + + // HU tax number: 8-1-2 digits (example: 12345678-1-23) + const normalized = normalizeTaxIdCompact(taxIdValue) + if (/^\d{8}-\d-\d{2}$/.test(normalized)) return 'hu_tin' + + return null +} + +/** + * Determine the Stripe tax ID type for Japan. + * + * Source reference: + * - Stripe docs list supported types + example formats (Japan section) + * https://docs.stripe.com/billing/customer/tax-ids + * + * @param {string|undefined|null} taxIdValue + * @returns {'jp_cn'|'jp_rn'|'jp_trn'|null} + */ +export function getJapanTaxIdType(taxIdValue) { + const normalized = normalizeTaxIdCompact(taxIdValue) + if (!normalized) return null + + // Tax Registration Number: T + 13 digits (example: T1234567891234) + if (/^T\d{13}$/.test(normalized)) return 'jp_trn' + + // Corporate Number: 13 digits (example: 1234567891234) + if (/^\d{13}$/.test(normalized)) return 'jp_cn' + + // Registered Foreign Businesses: 5 digits (example: 12345) + if (/^\d{5}$/.test(normalized)) return 'jp_rn' + + return null +} + +/** + * Determine the Stripe tax ID type for Liechtenstein (UID vs VAT). + * + * Source reference: + * - Stripe docs list supported types + example formats (Liechtenstein section) + * https://docs.stripe.com/billing/customer/tax-ids + * + * @param {string|undefined|null} taxIdValue + * @returns {'li_uid'|'li_vat'|null} + */ +export function getLiechtensteinTaxIdType(taxIdValue) { + const normalized = normalizeTaxIdCompact(taxIdValue) + if (!normalized) return null + + // UID: CHE + 9 digits (example: CHE123456789) + if (/^CHE\d{9}$/.test(normalized)) return 'li_uid' + + // VAT: 5 digits (example: 12345) + if (/^\d{5}$/.test(digitsOnly(normalized))) return 'li_vat' + + return null +} + +/** + * Determine the Stripe tax ID type for Malaysia (FRP, ITN, SST). + * + * Source reference: + * - Stripe docs list supported types + example formats (Malaysia section) + * https://docs.stripe.com/billing/customer/tax-ids + * + * @param {string|undefined|null} taxIdValue + * @returns {'my_frp'|'my_itn'|'my_sst'|null} + */ +export function getMalaysiaTaxIdType(taxIdValue) { + const normalized = normalizeTaxIdCompact(taxIdValue) + if (!normalized) return null + + // FRP: 8 digits (example: 12345678) + if (/^\d{8}$/.test(digitsOnly(normalized))) return 'my_frp' + + // ITN: letter + 10 digits (example: C 1234567890) + if (/^[A-Z]\d{10}$/.test(normalized.replace(/\s+/g, ''))) return 'my_itn' + + // SST: A12-3456-78912345 + if (/^[A-Z]\d{2}-\d{4}-\d{8}$/.test(normalized)) return 'my_sst' + + return null +} + +/** + * Determine the Stripe tax ID type for Norway (VAT vs VOEC). + * + * Source reference: + * - Stripe docs list supported types + example formats (Norway section) + * https://docs.stripe.com/billing/customer/tax-ids + * + * @param {string|undefined|null} taxIdValue + * @returns {'no_vat'|'no_voec'|null} + */ +export function getNorwayTaxIdType(taxIdValue) { + const normalized = normalizeTaxIdCompact(taxIdValue) + if (!normalized) return null + + // VAT: 9 digits + MVA (example: 123456789MVA) + if (/^\d{9}MVA$/.test(normalized)) return 'no_vat' + + // VOEC: 7 digits (example: 1234567) + if (/^\d{7}$/.test(digitsOnly(normalized))) return 'no_voec' + + return null +} + +/** + * Determine the Stripe tax ID type for Poland (EU VAT vs NIP). + * + * Source reference: + * - Stripe docs list supported types + example formats (Poland section) + * https://docs.stripe.com/billing/customer/tax-ids + * + * @param {string|undefined|null} taxIdValue + * @returns {'eu_vat'|'pl_nip'|null} + */ +export function getPolandTaxIdType(taxIdValue) { + if (!taxIdValue) return null + + // EU VAT: PL prefix (example: PL1234567890) + if (hasEuVatPrefix('PL', taxIdValue)) return 'eu_vat' + + // NIP: 10 digits (example: 1234567890) + if (/^\d{10}$/.test(digitsOnly(taxIdValue))) return 'pl_nip' + + return null +} + +/** + * Determine the Stripe tax ID type for Romania (EU VAT vs TIN). + * + * Source reference: + * - Stripe docs list supported types + example formats (Romania section) + * https://docs.stripe.com/billing/customer/tax-ids + * + * @param {string|undefined|null} taxIdValue + * @returns {'eu_vat'|'ro_tin'|null} + */ +export function getRomaniaTaxIdType(taxIdValue) { + if (!taxIdValue) return null + + // EU VAT: RO prefix (example: RO1234567891) + if (hasEuVatPrefix('RO', taxIdValue)) return 'eu_vat' + + // TIN: 13 digits (example: 1234567890123) + if (/^\d{13}$/.test(digitsOnly(taxIdValue))) return 'ro_tin' + + return null +} + +/** + * Determine the Stripe tax ID type for Russia (INN vs KPP). + * + * Source reference: + * - Stripe docs list supported types + example formats (Russia section) + * https://docs.stripe.com/billing/customer/tax-ids + * + * @param {string|undefined|null} taxIdValue + * @returns {'ru_inn'|'ru_kpp'|null} + */ +export function getRussiaTaxIdType(taxIdValue) { + const digits = digitsOnly(taxIdValue) + if (!digits) return null + + // INN: 10 digits (example: 1234567891) + if (/^\d{10}$/.test(digits)) return 'ru_inn' + + // KPP: 9 digits (example: 123456789) + if (/^\d{9}$/.test(digits)) return 'ru_kpp' + + return null +} + +/** + * Determine the Stripe tax ID type for Singapore (GST vs UEN). + * + * Source reference: + * - Stripe docs list supported types + example formats (Singapore section) + * https://docs.stripe.com/billing/customer/tax-ids + * + * @param {string|undefined|null} taxIdValue + * @returns {'sg_gst'|'sg_uen'|null} + */ +export function getSingaporeTaxIdType(taxIdValue) { + const normalized = normalizeTaxIdCompact(taxIdValue) + if (!normalized) return null + + // GST: M + 8 digits + letter (example: M12345678X) + if (/^M\d{8}[A-Z]$/.test(normalized)) return 'sg_gst' + + // UEN: 9 digits + letter (example: 123456789F) + if (/^\d{9}[A-Z]$/.test(normalized)) return 'sg_uen' + + return null +} + +/** + * Determine the Stripe tax ID type for Slovenia (EU VAT vs TIN). + * + * Source reference: + * - Stripe docs list supported types + example formats (Slovenia section) + * https://docs.stripe.com/billing/customer/tax-ids + * + * @param {string|undefined|null} taxIdValue + * @returns {'eu_vat'|'si_tin'|null} + */ +export function getSloveniaTaxIdType(taxIdValue) { + if (!taxIdValue) return null + + // EU VAT: SI prefix (example: SI12345678) + if (hasEuVatPrefix('SI', taxIdValue)) return 'eu_vat' + + // TIN: 8 digits (example: 12345678) + if (/^\d{8}$/.test(digitsOnly(taxIdValue))) return 'si_tin' + + return null +} + +/** + * Determine the Stripe tax ID type for Spain (EU VAT vs CIF/NIF). + * + * Source reference: + * - Stripe docs list supported types + example formats (Spain section) + * https://docs.stripe.com/billing/customer/tax-ids + * + * @param {string|undefined|null} taxIdValue + * @returns {'eu_vat'|'es_cif'|null} + */ +export function getSpainTaxIdType(taxIdValue) { + const normalized = normalizeTaxIdCompact(taxIdValue) + if (!normalized) return null + + // EU VAT: ES prefix (example: ESA1234567Z) + if (hasEuVatPrefix('ES', normalized)) return 'eu_vat' + + // CIF/NIF: A12345678 (letter + 7 digits + alnum) + if (/^[A-Z]\d{7}[A-Z0-9]$/.test(normalized)) return 'es_cif' + + return null +} + +/** + * Determine the Stripe tax ID type for Switzerland (UID vs VAT). + * + * Source reference: + * - Stripe docs list supported types + example formats (Switzerland section) + * https://docs.stripe.com/billing/customer/tax-ids + * + * @param {string|undefined|null} taxIdValue + * @returns {'ch_uid'|'ch_vat'|null} + */ +export function getSwitzerlandTaxIdType(taxIdValue) { + const normalized = normalizeTaxIdCompact(taxIdValue) + if (!normalized) return null + + const alnum = normalized.replace(/[^A-Z0-9]/g, '') + + // UID: CHE-123.456.789 HR + if (/^CHE\d{9}HR$/.test(alnum)) return 'ch_uid' + + // VAT: CHE-123.456.789 MWST + if (/^CHE\d{9}MWST$/.test(alnum)) return 'ch_vat' + + return null +} + +/** + * Determine the Stripe tax ID type for the United Kingdom (GB VAT vs EU VAT for NI). + * + * Source reference: + * - Stripe docs list supported types + example formats (UK section) + * https://docs.stripe.com/billing/customer/tax-ids + * + * @param {string|undefined|null} taxIdValue + * @returns {'gb_vat'|'eu_vat'|null} + */ +export function getUkTaxIdType(taxIdValue) { + const normalized = normalizeTaxIdCompact(taxIdValue) + if (!normalized) return null + + // Northern Ireland VAT numbers use XI prefix + if (/^XI[A-Z0-9]+$/.test(normalized)) return 'eu_vat' + + // GB VAT numbers use GB prefix + if (/^GB[A-Z0-9]+$/.test(normalized)) return 'gb_vat' + + return null +} + +/** + * Determine the Stripe tax ID type for Uzbekistan (TIN vs VAT). + * + * Source reference: + * - Stripe docs list supported types + example formats (Uzbekistan section) + * https://docs.stripe.com/billing/customer/tax-ids + * + * @param {string|undefined|null} taxIdValue + * @returns {'uz_tin'|'uz_vat'|null} + */ +export function getUzbekistanTaxIdType(taxIdValue) { + const digits = digitsOnly(taxIdValue) + if (!digits) return null + + // VAT: 12 digits (example: 123456789012) + if (/^\d{12}$/.test(digits)) return 'uz_vat' + + // TIN: 9 digits (example: 123456789) + if (/^\d{9}$/.test(digits)) return 'uz_tin' + + return null +} + /** * Get the Stripe tax ID type for a given country + tax ID value. * @@ -285,79 +747,152 @@ export function getTaxIdType(country, taxIdValue, postalCode) { const upperCountry = String(country).toUpperCase() + if (upperCountry === 'EU') { + // European One Stop Shop VAT number for non-Union scheme + const normalized = normalizeTaxIdCompact(taxIdValue) + return /^EU\d+$/.test(normalized) ? 'eu_oss_vat' : null + } + // EU VAT if (EU_VAT_COUNTRIES.includes(upperCountry)) { - return 'eu_vat' + // If this country has multiple types, we'll handle it below with a dedicated function. + // Otherwise, EU VAT is the only supported type for that country. + const euVatOnlyCountries = new Set([ + 'AT', + 'BE', + 'CY', + 'CZ', + 'DK', + 'EE', + 'FI', + 'FR', + 'GR', + 'IE', + 'IT', + 'LT', + 'LU', + 'LV', + 'MT', + 'NL', + 'PT', + 'SE', + 'SK', + ]) + if (euVatOnlyCountries.has(upperCountry)) return 'eu_vat' } - // Canada - if (upperCountry === 'CA') { - return getCanadaTaxIdType(taxIdValue, postalCode) - } + // Multi-type countries + if (upperCountry === 'CA') return getCanadaTaxIdType(taxIdValue, postalCode) + if (upperCountry === 'AU') return getAustraliaTaxIdType(taxIdValue) + if (upperCountry === 'BR') return getBrazilTaxIdType(taxIdValue) + if (upperCountry === 'BG') return getBulgariaTaxIdType(taxIdValue) + if (upperCountry === 'HR') return getCroatiaTaxIdType(taxIdValue) + if (upperCountry === 'DE') return getGermanyTaxIdType(taxIdValue) + if (upperCountry === 'HU') return getHungaryTaxIdType(taxIdValue) + if (upperCountry === 'JP') return getJapanTaxIdType(taxIdValue) + if (upperCountry === 'LI') return getLiechtensteinTaxIdType(taxIdValue) + if (upperCountry === 'MY') return getMalaysiaTaxIdType(taxIdValue) + if (upperCountry === 'NO') return getNorwayTaxIdType(taxIdValue) + if (upperCountry === 'PL') return getPolandTaxIdType(taxIdValue) + if (upperCountry === 'RO') return getRomaniaTaxIdType(taxIdValue) + if (upperCountry === 'RU') return getRussiaTaxIdType(taxIdValue) + if (upperCountry === 'SG') return getSingaporeTaxIdType(taxIdValue) + if (upperCountry === 'SI') return getSloveniaTaxIdType(taxIdValue) + if (upperCountry === 'ES') return getSpainTaxIdType(taxIdValue) + if (upperCountry === 'CH') return getSwitzerlandTaxIdType(taxIdValue) + if (upperCountry === 'GB') return getUkTaxIdType(taxIdValue) + if (upperCountry === 'UZ') return getUzbekistanTaxIdType(taxIdValue) // Country-specific tax IDs (all Stripe-supported types) - // See: https://docs.stripe.com/api/tax_ids/create#create_tax_id-type + // See: https://docs.stripe.com/billing/customer/tax-ids const countryTaxIdTypes = { - // Europe (non-EU) - GB: 'gb_vat', - // CH: 'ch_vat', - // NO: 'no_vat', - // IS: 'is_vat', - // LI: 'li_uid', - // TR: 'tr_tin', + // Africa + AO: 'ao_tin', + BH: 'bh_vat', + BF: 'bf_ifu', + BJ: 'bj_ifu', + CM: 'cm_niu', + CV: 'cv_nif', + CD: 'cd_nif', + EG: 'eg_tin', + ET: 'et_tin', + GN: 'gn_nif', + KE: 'ke_pin', + MA: 'ma_vat', + MR: 'mr_nif', + NG: 'ng_tin', + SN: 'sn_ninea', + TZ: 'tz_vat', + UG: 'ug_tin', + ZA: 'za_vat', + ZM: 'zm_tin', + ZW: 'zw_tin', - // // Americas + // Americas + AR: 'ar_cuit', + BO: 'bo_tin', + BS: 'bs_tin', + BB: 'bb_tin', + CL: 'cl_tin', + CO: 'co_nit', + CR: 'cr_tin', + DO: 'do_rcn', + EC: 'ec_ruc', + MX: 'mx_rfc', + PE: 'pe_ruc', + SR: 'sr_fin', + SV: 'sv_nit', US: 'us_ein', - // CA: 'ca_bn', // this is more complex, see getCanadaTaxIdType() - // MX: 'mx_rfc', - // BR: 'br_cnpj', - // CL: 'cl_tin', - // CO: 'co_nit', - // AR: 'ar_cuit', - // BO: 'bo_tin', - // CR: 'cr_tin', - // DO: 'do_rcn', - // EC: 'ec_ruc', - // PE: 'pe_ruc', - // UY: 'uy_ruc', - // VE: 've_rif', - // SV: 'sv_nit', + UY: 'uy_ruc', + VE: 've_rif', - // // Asia-Pacific - // AU: 'au_abn', - // NZ: 'nz_gst', - // JP: 'jp_cn', - // KR: 'kr_brn', - // CN: 'cn_tin', - // HK: 'hk_br', - // TW: 'tw_vat', - // SG: 'sg_gst', - // MY: 'my_sst', - // TH: 'th_vat', - // ID: 'id_npwp', - // PH: 'ph_tin', - // IN: 'in_gst', - // VN: 'vn_tin', + // Asia-Pacific + BD: 'bd_bin', + CN: 'cn_tin', + HK: 'hk_br', + ID: 'id_npwp', + IN: 'in_gst', + KH: 'kh_tin', + KR: 'kr_brn', + KZ: 'kz_bin', + KG: 'kg_tin', + LA: 'la_tin', + NZ: 'nz_gst', + NP: 'np_pan', + PH: 'ph_tin', + TH: 'th_vat', + TW: 'tw_vat', + VN: 'vn_tin', - // // Middle East - // AE: 'ae_trn', - // SA: 'sa_vat', - // BH: 'bh_vat', - // OM: 'om_vat', - // IL: 'il_vat', + // Europe (non-EU) + AD: 'ad_nrt', + AL: 'al_tin', + AM: 'am_tin', + AW: 'aw_tin', + AZ: 'az_tin', + BA: 'ba_tin', + BY: 'by_tin', + CH: 'ch_vat', + GE: 'ge_vat', + IS: 'is_vat', + LI: 'li_uid', + MD: 'md_vat', + ME: 'me_pib', + MK: 'mk_vat', + NO: 'no_vat', + RS: 'rs_pib', + RU: 'ru_inn', + TR: 'tr_tin', + UA: 'ua_vat', - // // Africa - // ZA: 'za_vat', - // EG: 'eg_tin', - // KE: 'ke_pin', - // NG: 'ng_tin', + // Middle East + AE: 'ae_trn', + IL: 'il_vat', + OM: 'om_vat', + SA: 'sa_vat', - // // Other - // GE: 'ge_vat', - // UA: 'ua_vat', - // RS: 'rs_pib', - // MD: 'md_vat', - // AD: 'ad_nrt', + // Other + EU: 'eu_oss_vat', } return countryTaxIdTypes[upperCountry] || null diff --git a/services/web/scripts/helpers/migrate_recurly_customers_to_stripe.helpers.node.test.mjs b/services/web/scripts/helpers/migrate_recurly_customers_to_stripe.helpers.node.test.mjs index 24ca1556ac..a3bc4e0872 100644 --- a/services/web/scripts/helpers/migrate_recurly_customers_to_stripe.helpers.node.test.mjs +++ b/services/web/scripts/helpers/migrate_recurly_customers_to_stripe.helpers.node.test.mjs @@ -16,6 +16,26 @@ import { coalesceOrEqualOrThrowName, coalesceOrThrowVATNumber, getCanadaTaxIdType, + getAustraliaTaxIdType, + getBrazilTaxIdType, + getBulgariaTaxIdType, + getCroatiaTaxIdType, + getGermanyTaxIdType, + getHungaryTaxIdType, + getJapanTaxIdType, + getLiechtensteinTaxIdType, + getMalaysiaTaxIdType, + getNorwayTaxIdType, + getPolandTaxIdType, + getRomaniaTaxIdType, + getRussiaTaxIdType, + getSingaporeTaxIdType, + getSloveniaTaxIdType, + getSpainTaxIdType, + getSwitzerlandTaxIdType, + getUkTaxIdType, + getUzbekistanTaxIdType, + getTaxIdType, coalesceOrThrowPaymentMethod, } from './migrate_recurly_customers_to_stripe.helpers.mjs' @@ -232,6 +252,108 @@ test('getCanadaTaxIdType returns null when format is unknown/ambiguous', () => { assert.equal(getCanadaTaxIdType('PST12345678', null), null) }) +test('getAustraliaTaxIdType distinguishes ABN vs ARN', () => { + assert.equal(getAustraliaTaxIdType('12345678912'), 'au_abn') + assert.equal(getAustraliaTaxIdType('123456789123'), 'au_arn') +}) + +test('getBrazilTaxIdType distinguishes CNPJ vs CPF', () => { + assert.equal(getBrazilTaxIdType('01.234.456/5432-10'), 'br_cnpj') + assert.equal(getBrazilTaxIdType('123.456.789-87'), 'br_cpf') +}) + +test('getBulgariaTaxIdType distinguishes EU VAT vs UIC', () => { + assert.equal(getBulgariaTaxIdType('BG0123456789'), 'eu_vat') + assert.equal(getBulgariaTaxIdType('123456789'), 'bg_uic') +}) + +test('getCroatiaTaxIdType distinguishes EU VAT vs OIB', () => { + assert.equal(getCroatiaTaxIdType('HR12345678912'), 'eu_vat') + assert.equal(getCroatiaTaxIdType('12345678901'), 'hr_oib') +}) + +test('getGermanyTaxIdType distinguishes EU VAT vs Steuer Nummer', () => { + assert.equal(getGermanyTaxIdType('DE123456789'), 'eu_vat') + assert.equal(getGermanyTaxIdType('1234567890'), 'de_stn') +}) + +test('getHungaryTaxIdType distinguishes EU VAT vs HU tax number', () => { + assert.equal(getHungaryTaxIdType('HU12345678'), 'eu_vat') + assert.equal(getHungaryTaxIdType('12345678-1-23'), 'hu_tin') +}) + +test('getJapanTaxIdType distinguishes TRN, CN, RN', () => { + assert.equal(getJapanTaxIdType('T1234567891234'), 'jp_trn') + assert.equal(getJapanTaxIdType('1234567891234'), 'jp_cn') + assert.equal(getJapanTaxIdType('12345'), 'jp_rn') +}) + +test('getLiechtensteinTaxIdType distinguishes UID vs VAT', () => { + assert.equal(getLiechtensteinTaxIdType('CHE123456789'), 'li_uid') + assert.equal(getLiechtensteinTaxIdType('12345'), 'li_vat') +}) + +test('getMalaysiaTaxIdType distinguishes FRP, ITN, SST', () => { + assert.equal(getMalaysiaTaxIdType('12345678'), 'my_frp') + assert.equal(getMalaysiaTaxIdType('C 1234567890'), 'my_itn') + assert.equal(getMalaysiaTaxIdType('A12-3456-78912345'), 'my_sst') +}) + +test('getNorwayTaxIdType distinguishes VAT vs VOEC', () => { + assert.equal(getNorwayTaxIdType('123456789MVA'), 'no_vat') + assert.equal(getNorwayTaxIdType('1234567'), 'no_voec') +}) + +test('getPolandTaxIdType distinguishes EU VAT vs NIP', () => { + assert.equal(getPolandTaxIdType('PL1234567890'), 'eu_vat') + assert.equal(getPolandTaxIdType('1234567890'), 'pl_nip') +}) + +test('getRomaniaTaxIdType distinguishes EU VAT vs TIN', () => { + assert.equal(getRomaniaTaxIdType('RO1234567891'), 'eu_vat') + assert.equal(getRomaniaTaxIdType('1234567890123'), 'ro_tin') +}) + +test('getRussiaTaxIdType distinguishes INN vs KPP', () => { + assert.equal(getRussiaTaxIdType('1234567891'), 'ru_inn') + assert.equal(getRussiaTaxIdType('123456789'), 'ru_kpp') +}) + +test('getSingaporeTaxIdType distinguishes GST vs UEN', () => { + assert.equal(getSingaporeTaxIdType('M12345678X'), 'sg_gst') + assert.equal(getSingaporeTaxIdType('123456789F'), 'sg_uen') +}) + +test('getSloveniaTaxIdType distinguishes EU VAT vs TIN', () => { + assert.equal(getSloveniaTaxIdType('SI12345678'), 'eu_vat') + assert.equal(getSloveniaTaxIdType('12345678'), 'si_tin') +}) + +test('getSpainTaxIdType distinguishes EU VAT vs CIF', () => { + assert.equal(getSpainTaxIdType('ESA1234567Z'), 'eu_vat') + assert.equal(getSpainTaxIdType('A12345678'), 'es_cif') +}) + +test('getSwitzerlandTaxIdType distinguishes UID vs VAT', () => { + assert.equal(getSwitzerlandTaxIdType('CHE-123.456.789 HR'), 'ch_uid') + assert.equal(getSwitzerlandTaxIdType('CHE-123.456.789 MWST'), 'ch_vat') +}) + +test('getUkTaxIdType distinguishes GB VAT vs EU VAT (NI)', () => { + assert.equal(getUkTaxIdType('GB123456789'), 'gb_vat') + assert.equal(getUkTaxIdType('XI123456789'), 'eu_vat') +}) + +test('getUzbekistanTaxIdType distinguishes TIN vs VAT', () => { + assert.equal(getUzbekistanTaxIdType('123456789'), 'uz_tin') + assert.equal(getUzbekistanTaxIdType('123456789012'), 'uz_vat') +}) + +test('getTaxIdType handles EU OSS VAT and EU VAT defaults', () => { + assert.equal(getTaxIdType('EU', 'EU123456789'), 'eu_oss_vat') + assert.equal(getTaxIdType('AT', 'ATU12345678'), 'eu_vat') +}) + test('coalesceOrThrowPaymentMethod throws when payment methods array is empty', () => { assert.throws( () => coalesceOrThrowPaymentMethod([], 'cus_123', {}), diff --git a/services/web/scripts/recurly/migrate_recurly_customers_to_stripe.mjs b/services/web/scripts/recurly/migrate_recurly_customers_to_stripe.mjs index 7603785e8e..e8483339da 100644 --- a/services/web/scripts/recurly/migrate_recurly_customers_to_stripe.mjs +++ b/services/web/scripts/recurly/migrate_recurly_customers_to_stripe.mjs @@ -61,6 +61,15 @@ * Options: * --input, -i Path to input CSV file * --output, -o Path to success output CSV file + * --limit, -l Limit number of records processed (default: no limit) + * --concurrency, -c Number of customers to process concurrently (default: 10) + * --recurly-rate-limit Requests per second for Recurly (default: 10) + * --recurly-api-retries Number of retries on Recurly 429s (default: 5) + * --recurly-retry-delay-ms Delay between Recurly retries in ms (default: 1000) + * --stripe-rate-limit Requests per second for Stripe (default: 50) + * --stripe-api-retries Number of retries on Stripe 429s (default: 5) + * --stripe-retry-delay-ms Delay between Stripe retries in ms (default: 1000) + * --force-invalid-tax Allow VAT numbers that cannot be mapped to a tax ID type (default: false) * --commit Actually update customers in Stripe (default: dry-run mode) * --verbose, -v Enable debug logging * --restart Ignore existing output files and start fresh @@ -69,9 +78,9 @@ * Note, prior to running this script, environment variables must have been loaded from config/local.env * * ``` - * set +a - * source ../../config/local.env * set -a + * source ../../config/local.env + * set +a * ``` */ @@ -79,6 +88,7 @@ import Settings from '@overleaf/settings' import Stripe from 'stripe' import recurly from 'recurly' import minimist from 'minimist' +import PQueue from 'p-queue' import fs from 'node:fs' import * as csv from 'csv' import { setTimeout } from 'node:timers/promises' @@ -159,6 +169,66 @@ if (!recurlyApiKey) { } const recurlyClient = new recurly.Client(recurlyApiKey) +// ============================================================================= +// CUSTOM FIELDS +// ============================================================================= + +const RECURLY_CUSTOM_FIELD_NAMES = [ + 'channel', + 'Industry', + 'ol_sales_person', + 'MigratedfromFreeAgent', +] + +function getRecurlyCustomFields(account) { + const customFields = account?.customFields + if (customFields == null) { + throw new Error( + 'Recurly account is missing customFields (empty array is acceptable)' + ) + } + if (!Array.isArray(customFields)) { + throw new Error('Recurly account customFields is not an array') + } + return customFields +} + +function extractRecurlyCustomFieldMetadata(account) { + const customFields = getRecurlyCustomFields(account) + + /** @type {Record} */ + const metadata = {} + + const counts = { + channel: 0, + Industry: 0, + ol_sales_person: 0, + MigratedfromFreeAgent: 0, + noCustomFields: 0, + } + + if (customFields.length === 0) { + counts.noCustomFields = 1 + return { metadata, counts } + } + + for (const field of customFields) { + const name = field?.name?.trim() + if (!RECURLY_CUSTOM_FIELD_NAMES.includes(name)) continue + + const rawValue = field?.value + if (rawValue == null) continue + + const value = String(rawValue).trim() + if (!value) continue + + metadata[name] = value + counts[name] = 1 + } + + return { metadata, counts } +} + // ============================================================================= // LOGGING UTILITIES // ============================================================================= @@ -241,7 +311,15 @@ async function loadSuccessfullyProcessed(successOutputPath) { return new Promise((resolve, reject) => { fs.createReadStream(successOutputPath) - .pipe(csv.parse({ columns: true, trim: true })) + .pipe( + csv.parse({ + columns: true, + trim: true, + skip_empty_lines: true, + relax_column_count: true, + relax_column_count_less: true, + }) + ) .on('data', row => { if (row.recurly_account_code) { processed.add(row.recurly_account_code) @@ -534,12 +612,34 @@ class RateLimiter { } } -// Recurly: 2000 requests per 5 minutes, target 1500 (75% of limit) for safety margin -const recurlyRateLimiter = new RateLimiter('Recurly', 1500, 5 * 60 * 1000) +const DEFAULT_RECURLY_RATE_LIMIT = 10 // requests per second +const DEFAULT_STRIPE_RATE_LIMIT = 50 // requests per second +const DEFAULT_RECURLY_API_RETRIES = 5 +const DEFAULT_RECURLY_RETRY_DELAY_MS = 1000 +const DEFAULT_STRIPE_API_RETRIES = 5 +const DEFAULT_STRIPE_RETRY_DELAY_MS = 1000 +const RATE_LIMIT_WINDOW_MS = 1000 -// Stripe: 100 requests per second, target 50 (50% of limit) - much more headroom -// Using 10-second window with 500 requests = 50/sec average -const stripeRateLimiter = new RateLimiter('Stripe', 500, 10 * 1000) +let recurlyRateLimiter +let recurlyApiRetries = DEFAULT_RECURLY_API_RETRIES +let recurlyRetryDelayMs = DEFAULT_RECURLY_RETRY_DELAY_MS +let stripeRateLimitPerSecond = DEFAULT_STRIPE_RATE_LIMIT +let stripeApiRetries = DEFAULT_STRIPE_API_RETRIES +let stripeRetryDelayMs = DEFAULT_STRIPE_RETRY_DELAY_MS +const stripeRateLimiters = new Map() + +function getStripeRateLimiter(region) { + const key = String(region || 'unknown').toLowerCase() + if (stripeRateLimiters.has(key)) return stripeRateLimiters.get(key) + + const limiter = new RateLimiter( + `Stripe-${key}`, + stripeRateLimitPerSecond, + RATE_LIMIT_WINDOW_MS + ) + stripeRateLimiters.set(key, limiter) + return limiter +} /** * Throttle before making a Recurly API call @@ -548,23 +648,125 @@ async function throttleRecurly() { await recurlyRateLimiter.throttle() } +async function recurlyRequestWithRetries(operation, { context } = {}) { + let attempt = 0 + while (true) { + try { + return await operation() + } catch (error) { + const statusCode = + error?.statusCode ?? error?.status ?? error?.raw?.statusCode + if (statusCode === 429) { + logWarn('Recurly rate limited', { + rowNumber: context?.rowNumber, + recurlyAccountCode: context?.recurlyAccountCode, + attempt: attempt + 1, + maxRetries: recurlyApiRetries, + }) + + if (attempt < recurlyApiRetries) { + attempt++ + await setTimeout(recurlyRetryDelayMs) + continue + } + } + throw error + } + } +} + +async function recurlyCall(operation, context) { + return recurlyRequestWithRetries( + async () => { + await throttleRecurly() + return operation() + }, + { context } + ) +} + /** * Throttle before making a Stripe API call */ -async function throttleStripe() { - await stripeRateLimiter.throttle() +async function throttleStripe(region) { + await getStripeRateLimiter(region).throttle() } /** * Get rate limiter statistics for logging */ function getRateLimiterStats() { + const stripeLimiters = [...stripeRateLimiters.values()] + const stripeTotalRequests = stripeLimiters.reduce( + (sum, limiter) => sum + limiter.totalRequests, + 0 + ) + const stripeCurrentRate = stripeLimiters.reduce( + (sum, limiter) => sum + limiter.getCurrentRate(), + 0 + ) + return { recurly: recurlyRateLimiter.getStats(), - stripe: stripeRateLimiter.getStats(), + stripe: { + totalRequests: stripeTotalRequests, + currentRate: stripeCurrentRate.toFixed(2) + '/sec', + }, + stripeByRegion: stripeLimiters.map(limiter => limiter.getStats()), } } +function getStripeRateLimitReason(error) { + const headers = + error?.headers || error?.raw?.headers || error?.response?.headers || {} + return ( + headers['stripe-rate-limit-reason'] || + headers['Stripe-Rate-Limited-Reason'] || + headers['stripe-rate-limited-reason'] || + null + ) +} + +async function stripeRequestWithRetries(operation, { context } = {}) { + let attempt = 0 + while (true) { + try { + return await operation() + } catch (error) { + const statusCode = error?.statusCode ?? error?.raw?.statusCode + if (statusCode === 429) { + const reason = getStripeRateLimitReason(error) + logWarn('Stripe rate limited', { + rowNumber: context?.rowNumber, + stripeCustomerId: context?.stripeCustomerId, + stripeAccount: context?.stripeAccount, + reason, + stripeApi: context?.stripeApi, + attempt: attempt + 1, + maxRetries: stripeApiRetries, + }) + + if (attempt < stripeApiRetries) { + attempt++ + await setTimeout(stripeRetryDelayMs) + continue + } + } + throw error + } + } +} + +async function stripeCall(region, operation, context) { + return stripeRequestWithRetries( + async () => { + await throttleStripe(region) + return operation() + }, + { context } + ) +} + // ============================================================================= // DATA TRANSFORMATION // ============================================================================= @@ -575,14 +777,18 @@ function getRateLimiterStats() { * @param {string} accountCode - The Recurly account code (Overleaf user ID) * @returns {Promise<{account: object, billingInfo: object|null}>} */ -async function fetchRecurlyData(accountCode) { - await throttleRecurly() - const account = await recurlyClient.getAccount(`code-${accountCode}`) +async function fetchRecurlyData(accountCode, context) { + const account = await recurlyCall( + () => recurlyClient.getAccount(`code-${accountCode}`), + context + ) let billingInfo = null try { - await throttleRecurly() - billingInfo = await recurlyClient.getBillingInfo(`code-${accountCode}`) + billingInfo = await recurlyCall( + () => recurlyClient.getBillingInfo(`code-${accountCode}`), + context + ) } catch (error) { // Billing info may not exist for manually billed customers if (error instanceof recurly.errors.NotFoundError) { @@ -603,9 +809,18 @@ async function fetchRecurlyData(accountCode) { * @returns {Promise} * @throws {Error} If customer is not found or is deleted */ -async function fetchTargetStripeCustomer(stripeClient, stripeCustomerId) { - await throttleStripe() - const customer = await stripeClient.customers.retrieve(stripeCustomerId) +async function fetchTargetStripeCustomer( + stripeClient, + stripeCustomerId, + region, + context +) { + // TODO: consider getting the region from stripeClient.serviceName + const customer = await stripeCall( + region, + () => stripeClient.customers.retrieve(stripeCustomerId), + { ...context, stripeApi: 'customers.retrieve' } + ) if (customer.deleted) { throw new Error(`Stripe customer ${stripeCustomerId} has been deleted`) @@ -623,11 +838,15 @@ async function fetchTargetStripeCustomer(stripeClient, stripeCustomerId) { */ async function fetchTargetStripeCustomerPaymentMethods( stripeClient, - stripeCustomerId + stripeCustomerId, + region, + context ) { - await throttleStripe() - const paymentMethods = - await stripeClient.customers.listPaymentMethods(stripeCustomerId) + const paymentMethods = await stripeCall( + region, + () => stripeClient.customers.listPaymentMethods(stripeCustomerId), + { ...context, stripeApi: 'customers.listPaymentMethods' } + ) return paymentMethods.data } @@ -641,7 +860,8 @@ async function replaceCustomerTaxIds( stripeClient, stripeCustomerId, { taxIdType, vatNumber }, - context + context, + region ) { // Stripe customers can have multiple tax IDs. For this migration, we want a single // authoritative tax ID derived from Recurly, so we remove any existing ones first. @@ -649,11 +869,15 @@ async function replaceCustomerTaxIds( let startingAfter while (true) { - await throttleStripe() - const page = await stripeClient.customers.listTaxIds(stripeCustomerId, { - limit: 100, - ...(startingAfter ? { starting_after: startingAfter } : {}), - }) + const page = await stripeCall( + region, + () => + stripeClient.customers.listTaxIds(stripeCustomerId, { + limit: 100, + ...(startingAfter ? { starting_after: startingAfter } : {}), + }), + { ...context, stripeApi: 'customers.listTaxIds' } + ) existingTaxIds.push(...page.data) @@ -672,16 +896,23 @@ async function replaceCustomerTaxIds( ) for (const taxId of existingTaxIds) { - await throttleStripe() - await stripeClient.customers.deleteTaxId(stripeCustomerId, taxId.id) + await stripeCall( + region, + () => stripeClient.customers.deleteTaxId(stripeCustomerId, taxId.id), + { ...context, stripeApi: 'customers.deleteTaxId' } + ) } } - await throttleStripe() - return await stripeClient.customers.createTaxId(stripeCustomerId, { - type: taxIdType, - value: vatNumber, - }) + return await stripeCall( + region, + () => + stripeClient.customers.createTaxId(stripeCustomerId, { + type: taxIdType, + value: vatNumber, + }), + { ...context, stripeApi: 'customers.createTaxId' } + ) } /** @@ -767,7 +998,7 @@ async function processCustomer( row, rowNumber, commit, - { writeStripeExistingFields } = {} + { writeStripeExistingFields, forceInvalidTax = false } = {} ) { const { recurly_account_code: recurlyAccountCode, @@ -782,6 +1013,12 @@ async function processCustomer( stripeCustomerId, } + const stripeContext = { + rowNumber, + stripeCustomerId, + stripeAccount: targetStripeAccount, + } + const result = { recurly_account_code: recurlyAccountCode, target_stripe_account: targetStripeAccount, @@ -789,6 +1026,7 @@ async function processCustomer( outcome: '', // 'updated', 'dry_run', 'skipped_no_stripe_id', or 'error' error: '', customerParams: null, // Stripe customer params (for dry-run output) + taxInfoPending: null, // Recurly VAT number if tax ID type couldn't be determined } try { @@ -835,7 +1073,10 @@ async function processCustomer( { ...context, step: 'fetch_recurly' }, { verboseOnly: true } ) - const { account, billingInfo } = await fetchRecurlyData(recurlyAccountCode) + const { account, billingInfo } = await fetchRecurlyData( + recurlyAccountCode, + context + ) logDebug( 'Fetched Recurly account', @@ -895,7 +1136,9 @@ async function processCustomer( ) const existingCustomer = await fetchTargetStripeCustomer( stripeClient, - stripeCustomerId + stripeCustomerId, + region, + stripeContext ) logDebug( @@ -913,38 +1156,57 @@ async function processCustomer( let taxIdType = null let country = null let createdTaxId = null + let taxInfoPendingValue = null - // Validate VAT number can be created if present + // Determine VAT number tax ID type (if possible) if (vatNumber) { // We need to extract address first to get the country const tempAddress = coalesceOrEqualOrThrowAddress(account, billingInfo) country = tempAddress?.country if (!country) { - throw new Error( - `Customer has VAT number (${vatNumber}) but no country in address - cannot determine tax ID type` - ) - } - taxIdType = getTaxIdType(country, vatNumber, tempAddress?.postal_code) - if (!taxIdType) { - throw new Error( - `Unable to determine tax id type for (${vatNumber}), country ${country}, postal_code ${tempAddress?.postal_code}` - ) - } - logDebug( - 'Will create tax ID', - { + if (!forceInvalidTax) { + throw new Error(`Unprocessable VAT number ${vatNumber} (no country)`) + } + logWarn('VAT number present but no country in address', { ...context, vatNumber, - country, - taxIdType, - }, - { verboseOnly: true } - ) + }) + taxInfoPendingValue = vatNumber + } else { + taxIdType = getTaxIdType(country, vatNumber, tempAddress?.postal_code) + if (!taxIdType) { + if (!forceInvalidTax) { + throw new Error( + `Unprocessable VAT number ${vatNumber} (failed getTaxIdType)` + ) + } + logWarn('Unable to determine tax id type for VAT number', { + ...context, + vatNumber, + country, + postalCode: tempAddress?.postal_code, + }) + taxInfoPendingValue = vatNumber + } else { + logDebug( + 'Will create tax ID', + { + ...context, + vatNumber, + country, + taxIdType, + }, + { verboseOnly: true } + ) + } + } } + const shouldCreateTaxId = !!(vatNumber && taxIdType && !taxInfoPendingValue) + if (commit) { // Create tax ID first (validate it works before updating customer) - if (vatNumber && taxIdType) { + if (shouldCreateTaxId) { logDebug( 'Creating tax ID', { @@ -960,7 +1222,8 @@ async function processCustomer( stripeClient, stripeCustomerId, { taxIdType, vatNumber }, - context + context, + region ) logDebug( 'Successfully created tax ID', @@ -1010,7 +1273,9 @@ async function processCustomer( const paymentMethods = await fetchTargetStripeCustomerPaymentMethods( stripeClient, - stripeCustomerId + stripeCustomerId, + region, + stripeContext ) const paymentMethod = coalesceOrThrowPaymentMethod( paymentMethods, @@ -1023,6 +1288,9 @@ async function processCustomer( if (account.createdAt) { metadata.recurlyCreatedAt = account.createdAt.toISOString() } + if (taxInfoPendingValue) { + metadata.taxInfoPending = taxInfoPendingValue + } if ( existingCustomer.metadata != null && existingCustomer.metadata.recurlyAccountCode === recurlyAccountCode && @@ -1037,6 +1305,15 @@ async function processCustomer( }) } + const { metadata: customFieldMetadata, counts: customFieldCounts } = + extractRecurlyCustomFieldMetadata(account) + + if (Object.keys(customFieldMetadata).length > 0) { + Object.assign(metadata, customFieldMetadata) + } + + result.customFieldCounts = customFieldCounts + /** @type {Stripe.CustomerUpdateParams} */ const customerParams = { email: account.email, @@ -1135,8 +1412,11 @@ async function processCustomer( }, { verboseOnly: true } ) - await throttleStripe() - await stripeClient.customers.update(stripeCustomerId, customerParams) + await stripeCall( + region, + () => stripeClient.customers.update(stripeCustomerId, customerParams), + { ...stripeContext, stripeApi: 'customers.update' } + ) result.outcome = 'updated' logDebug( @@ -1151,7 +1431,7 @@ async function processCustomer( result.customerParams = { ...customerParams, // Include tax ID info in dry-run output for review - _taxId: vatNumber + _taxId: shouldCreateTaxId ? { type: taxIdType, value: vatNumber, @@ -1170,6 +1450,10 @@ async function processCustomer( { verboseOnly: true } ) } + + if (taxInfoPendingValue) { + result.taxInfoPending = taxInfoPendingValue + } } catch (error) { result.outcome = 'error' // Include more error details @@ -1204,6 +1488,33 @@ function usage() { console.error( ' --output, -o Path to SUCCESS output CSV file (required)' ) + console.error( + ' --limit, -l Limit number of records processed (default: no limit)' + ) + console.error( + ' --concurrency, -c Number of customers to process concurrently (default: 10)' + ) + console.error( + ' --recurly-rate-limit Requests per second for Recurly (default: 10)' + ) + console.error( + ' --recurly-api-retries Number of retries on Recurly 429s (default: 5)' + ) + console.error( + ' --recurly-retry-delay-ms Delay between Recurly retries in ms (default: 1000)' + ) + console.error( + ' --stripe-rate-limit Requests per second for Stripe (default: 50)' + ) + console.error( + ' --stripe-api-retries Number of retries on Stripe 429s (default: 5)' + ) + console.error( + ' --stripe-retry-delay-ms Delay between Stripe retries in ms (default: 1000)' + ) + console.error( + ' --force-invalid-tax Allow VAT numbers that cannot be mapped to a tax ID type (default: false)' + ) console.error( ' --commit Actually update customers in Stripe (default: dry-run)' ) @@ -1261,16 +1572,91 @@ function usage() { ) } +function parseConcurrency(value, { defaultValue = 10 } = {}) { + if (value === undefined || value === null || value === '') { + return defaultValue + } + + const parsed = Number(value) + if (!Number.isFinite(parsed) || !Number.isInteger(parsed) || parsed < 1) { + throw new Error( + `Invalid --concurrency value: ${value}. Expected a positive integer.` + ) + } + + return parsed +} + +function parseRateLimit(value, { defaultValue, name }) { + if (value === undefined || value === null || value === '') { + return defaultValue + } + + const parsed = Number(value) + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error( + `Invalid --${name} value: ${value}. Expected a positive number.` + ) + } + + return parsed +} + +function parseNonNegativeInt(value, { defaultValue, name }) { + if (value === undefined || value === null || value === '') { + return defaultValue + } + + const parsed = Number(value) + if (!Number.isFinite(parsed) || !Number.isInteger(parsed) || parsed < 0) { + throw new Error( + `Invalid --${name} value: ${value}. Expected a non-negative integer.` + ) + } + + return parsed +} + function parseArgs() { return minimist(process.argv.slice(2), { - alias: { i: 'input', o: 'output', h: 'help', v: 'verbose' }, - string: ['input', 'output'], - boolean: ['commit', 'verbose', 'help', 'restart'], - default: { commit: false, verbose: false, restart: false }, + alias: { + i: 'input', + o: 'output', + h: 'help', + v: 'verbose', + c: 'concurrency', + l: 'limit', + }, + string: [ + 'input', + 'output', + 'limit', + 'recurly-rate-limit', + 'recurly-api-retries', + 'recurly-retry-delay-ms', + 'stripe-rate-limit', + 'stripe-api-retries', + 'stripe-retry-delay-ms', + ], + boolean: ['commit', 'verbose', 'help', 'restart', 'force-invalid-tax'], + default: { + commit: false, + verbose: false, + restart: false, + 'force-invalid-tax': false, + concurrency: 10, + 'recurly-rate-limit': DEFAULT_RECURLY_RATE_LIMIT, + 'recurly-api-retries': DEFAULT_RECURLY_API_RETRIES, + 'recurly-retry-delay-ms': DEFAULT_RECURLY_RETRY_DELAY_MS, + 'stripe-rate-limit': DEFAULT_STRIPE_RATE_LIMIT, + 'stripe-api-retries': DEFAULT_STRIPE_API_RETRIES, + 'stripe-retry-delay-ms': DEFAULT_STRIPE_RETRY_DELAY_MS, + }, }) } async function main(trackProgress) { + const startTime = new Date() const args = parseArgs() const { input: inputPath, @@ -1279,8 +1665,71 @@ async function main(trackProgress) { verbose, help, restart, + 'force-invalid-tax': forceInvalidTax, + concurrency: concurrencyRaw, + limit: limitRaw, + 'recurly-rate-limit': recurlyRateLimitRaw, + 'recurly-api-retries': recurlyApiRetriesRaw, + 'recurly-retry-delay-ms': recurlyRetryDelayMsRaw, + 'stripe-rate-limit': stripeRateLimitRaw, + 'stripe-api-retries': stripeApiRetriesRaw, + 'stripe-retry-delay-ms': stripeRetryDelayMsRaw, } = args + let concurrency + let recurlyRateLimit + let recurlyApiRetriesValue + let recurlyRetryDelayMsValue + let stripeApiRetriesValue + let stripeRetryDelayMsValue + let limit + try { + concurrency = parseConcurrency(concurrencyRaw, { defaultValue: 10 }) + limit = parseNonNegativeInt(limitRaw, { + defaultValue: null, + name: 'limit', + }) + recurlyRateLimit = parseRateLimit(recurlyRateLimitRaw, { + defaultValue: DEFAULT_RECURLY_RATE_LIMIT, + name: 'recurly-rate-limit', + }) + recurlyApiRetriesValue = parseNonNegativeInt(recurlyApiRetriesRaw, { + defaultValue: DEFAULT_RECURLY_API_RETRIES, + name: 'recurly-api-retries', + }) + recurlyRetryDelayMsValue = parseNonNegativeInt(recurlyRetryDelayMsRaw, { + defaultValue: DEFAULT_RECURLY_RETRY_DELAY_MS, + name: 'recurly-retry-delay-ms', + }) + stripeRateLimitPerSecond = parseRateLimit(stripeRateLimitRaw, { + defaultValue: DEFAULT_STRIPE_RATE_LIMIT, + name: 'stripe-rate-limit', + }) + stripeApiRetriesValue = parseNonNegativeInt(stripeApiRetriesRaw, { + defaultValue: DEFAULT_STRIPE_API_RETRIES, + name: 'stripe-api-retries', + }) + stripeRetryDelayMsValue = parseNonNegativeInt(stripeRetryDelayMsRaw, { + defaultValue: DEFAULT_STRIPE_RETRY_DELAY_MS, + name: 'stripe-retry-delay-ms', + }) + } catch (error) { + logError(error.message) + usage() + process.exit(1) + } + + recurlyRateLimiter = new RateLimiter( + 'Recurly', + recurlyRateLimit, + RATE_LIMIT_WINDOW_MS + ) + recurlyApiRetries = recurlyApiRetriesValue + recurlyRetryDelayMs = recurlyRetryDelayMsValue + stripeApiRetries = stripeApiRetriesValue + stripeRetryDelayMs = stripeRetryDelayMsValue + stripeRateLimiters.clear() + // Set DEBUG_MODE only from CLI arg (--verbose/-v) DEBUG_MODE = !!verbose @@ -1303,6 +1752,15 @@ async function main(trackProgress) { skippedOutputPath, ...(commit ? {} : { stripeJsonPath }), stripeExistingFieldsJsonPath, + concurrency, + recurlyRateLimit, + recurlyApiRetries, + recurlyRetryDelayMs, + stripeRateLimit: stripeRateLimitPerSecond, + stripeApiRetries, + stripeRetryDelayMs, + forceInvalidTax, + ...(limit != null ? { limit } : {}), }) await trackProgress(`Starting migration in ${mode}`) @@ -1367,11 +1825,21 @@ async function main(trackProgress) { // Statistics let totalInInput = 0 let processedThisRun = 0 + let queuedThisRun = 0 let skippedPreviouslyProcessed = 0 let updatedCount = 0 let skippedNoStripeIdCount = 0 let errorCount = 0 let dryRunCount = 0 + let taxInfoPendingCount = 0 + + const customFieldStats = { + channel: 0, + Industry: 0, + ol_sales_person: 0, + MigratedfromFreeAgent: 0, + noCustomFields: 0, + } // Track errors for final summary (just the account codes, not full results - memory efficient) const errorAccountCodes = [] @@ -1380,84 +1848,152 @@ async function main(trackProgress) { // Process input CSV - true streaming (no collecting results in memory) const inputStream = fs.createReadStream(inputPath) - const parser = csv.parse({ columns: true, trim: true }) + const parser = csv.parse({ + columns: true, + trim: true, + skip_empty_lines: true, + relax_column_count: true, + relax_column_count_less: true, + }) inputStream.pipe(parser) + const queue = new PQueue({ concurrency }) + const maxQueueSize = concurrency + let lastCompletedRowNumber = 0 + let limitReached = false + let rowNumber = 0 - for await (const row of parser) { - rowNumber++ - totalInInput++ + try { + for await (const row of parser) { + rowNumber++ + totalInInput++ - const accountCode = row.recurly_account_code + const thisRowNumber = rowNumber + const accountCode = row.recurly_account_code - // Check if already successfully processed in a previous run - if (previouslyProcessed.has(accountCode)) { - skippedPreviouslyProcessed++ - logDebug( - 'Skipping previously successful record', - { - rowNumber, - accountCode, - }, - { verboseOnly: true } - ) - continue - } + // Check if already successfully processed in a previous run + if (previouslyProcessed.has(accountCode)) { + skippedPreviouslyProcessed++ + logDebug( + 'Skipping previously successful record', + { + rowNumber: thisRowNumber, + accountCode, + }, + { verboseOnly: true } + ) + continue + } - // Process this customer - const result = await processCustomer(row, rowNumber, commit, { - writeStripeExistingFields: stripeExistingFieldsWriter.write, - }) + if (limit != null && queuedThisRun >= limit) { + limitReached = true + logDebug('Record limit reached, stopping input processing', { + limit, + queuedThisRun, + rowNumber: thisRowNumber, + }) + break + } - processedThisRun++ + if (queue.size >= maxQueueSize) { + await queue.onSizeLessThan(maxQueueSize) + } - // Write to appropriate output file based on outcome - if (result.outcome === 'error') { - writeError(result) - errorCount++ - errorAccountCodes.push(accountCode) - } else if (result.outcome === 'skipped_no_stripe_id') { - writeSkipped(result) - skippedNoStripeIdCount++ - } else { - writeSuccess(result) - // Update statistics and collect dry-run data - if (result.outcome === 'updated') { - updatedCount++ - } else if (result.outcome === 'dry_run') { - dryRunCount++ - // Collect customer params for stripe.json output - if (result.customerParams) { - stripeCustomerParams.push({ - recurly_account_code: result.recurly_account_code, - target_stripe_account: result.target_stripe_account, - customerParams: result.customerParams, + queuedThisRun++ + queue.add(async () => { + let result + try { + result = await processCustomer(row, thisRowNumber, commit, { + writeStripeExistingFields: stripeExistingFieldsWriter.write, + forceInvalidTax, + }) + } catch (error) { + result = { + ...row, + outcome: 'error', + error: error?.message || String(error), + } + logError('Unhandled error while processing customer', error, { + rowNumber: thisRowNumber, + accountCode, }) } - } - } - // Progress update every 1000 customers (or 100 in debug mode) - const progressInterval = DEBUG_MODE ? 100 : 1000 - if (processedThisRun % progressInterval === 0) { - const rateLimiterStats = getRateLimiterStats() - const progress = { - rowNumber, - processedThisRun, - updated: updatedCount, - dryRun: dryRunCount, - skippedNoStripeId: skippedNoStripeIdCount, - errors: errorCount, - skippedPrevious: skippedPreviouslyProcessed, - recurlyRate: rateLimiterStats.recurly.currentRate, - stripeRate: rateLimiterStats.stripe.currentRate, - } - logDebug('Progress update', progress) - await trackProgress( - `Progress: row ${rowNumber}, ${processedThisRun} processed this run, ${errorCount} errors` - ) + processedThisRun++ + lastCompletedRowNumber = thisRowNumber + + if (result.customFieldCounts) { + for (const [field, count] of Object.entries( + result.customFieldCounts + )) { + if (customFieldStats[field] != null) { + customFieldStats[field] += count + } + } + } + + if (result.taxInfoPending != null) { + taxInfoPendingCount++ + } + + // Write to appropriate output file based on outcome + if (result.outcome === 'error') { + writeError(result) + errorCount++ + errorAccountCodes.push(accountCode) + } else if (result.outcome === 'skipped_no_stripe_id') { + writeSkipped(result) + skippedNoStripeIdCount++ + } else { + writeSuccess(result) + // Update statistics and collect dry-run data + if (result.outcome === 'updated') { + updatedCount++ + } else if (result.outcome === 'dry_run') { + dryRunCount++ + // Collect customer params for stripe.json output + if (result.customerParams) { + stripeCustomerParams.push({ + recurly_account_code: result.recurly_account_code, + target_stripe_account: result.target_stripe_account, + customerParams: result.customerParams, + }) + } + } + } + + // Progress update every 1000 customers (or 100 in debug mode) + const progressInterval = DEBUG_MODE ? 100 : 1000 + if (processedThisRun % progressInterval === 0) { + const rateLimiterStats = getRateLimiterStats() + const progress = { + rowNumber: lastCompletedRowNumber, + processedThisRun, + updated: updatedCount, + dryRun: dryRunCount, + skippedNoStripeId: skippedNoStripeIdCount, + taxInfoPending: taxInfoPendingCount, + errors: errorCount, + skippedPrevious: skippedPreviouslyProcessed, + recurlyRate: rateLimiterStats.recurly.currentRate, + stripeRate: rateLimiterStats.stripe.currentRate, + } + logDebug('Progress update', progress) + await trackProgress( + `Progress: row ${lastCompletedRowNumber}, ${processedThisRun} processed this run, ${errorCount} errors` + ) + } + }) } + } finally { + await queue.onIdle() + } + + if (limitReached) { + await trackProgress( + `Limit reached (${limit}). Stopped reading input; waiting for in-flight records to finish.` + ) } // Write stripe.json file in dry-run mode @@ -1472,30 +2008,43 @@ async function main(trackProgress) { } // Final summary + const endTime = new Date() + const durationMs = endTime.getTime() - startTime.getTime() + const durationTotalSeconds = Math.floor(durationMs / 1000) + const durationHours = Math.floor(durationTotalSeconds / 3600) + const durationMinutes = Math.floor((durationTotalSeconds % 3600) / 60) + const durationSeconds = durationTotalSeconds % 60 + const durationHms = + String(durationHours).padStart(2, '0') + + ':' + + String(durationMinutes).padStart(2, '0') + + ':' + + String(durationSeconds).padStart(2, '0') + const totalSuccessful = commit ? previouslyProcessed.size + updatedCount : previouslyProcessed.size const finalRateLimiterStats = getRateLimiterStats() - logDebug('=== FINAL SUMMARY ===') - logDebug(`Input file total rows: ${totalInInput}`) - logDebug(`Previously successful (skipped): ${skippedPreviouslyProcessed}`) - logDebug(`Processed this run: ${processedThisRun}`) - logDebug( - ` - ${commit ? 'Updated' : 'Would update'}: ${commit ? updatedCount : dryRunCount}` - ) - logDebug(` - Skipped (no stripe_customer_id): ${skippedNoStripeIdCount}`) - logDebug(` - Errors: ${errorCount}`) - if (commit) { - logDebug(`Total in success file: ${totalSuccessful}`) - } - logDebug(`Total in skipped file: ${skippedNoStripeIdCount}`) - logDebug(`Total in errors file: ${errorCount}`) - logDebug( - `API calls - Recurly: ${finalRateLimiterStats.recurly.totalRequests}, Stripe: ${finalRateLimiterStats.stripe.totalRequests}` - ) - await trackProgress('=== FINAL SUMMARY ===') + await trackProgress(`Start time: ${startTime.toISOString()}`) + await trackProgress(`End time: ${endTime.toISOString()}`) + await trackProgress(`Total runtime: ${durationHms}`) + await trackProgress('CLI parameters:') + await trackProgress(` - input: ${inputPath}`) + await trackProgress(` - output: ${successOutputPath}`) + await trackProgress(` - commit: ${commit}`) + await trackProgress(` - verbose: ${verbose}`) + await trackProgress(` - restart: ${restart}`) + await trackProgress(` - limit: ${limit != null ? limit : 'none'}`) + await trackProgress(` - concurrency: ${concurrency}`) + await trackProgress(` - recurly-rate-limit: ${recurlyRateLimit}`) + await trackProgress(` - recurly-api-retries: ${recurlyApiRetries}`) + await trackProgress(` - recurly-retry-delay-ms: ${recurlyRetryDelayMs}`) + await trackProgress(` - stripe-rate-limit: ${stripeRateLimitPerSecond}`) + await trackProgress(` - stripe-api-retries: ${stripeApiRetries}`) + await trackProgress(` - stripe-retry-delay-ms: ${stripeRetryDelayMs}`) + await trackProgress(` - force-invalid-tax: ${forceInvalidTax}`) await trackProgress(`Input file total rows: ${totalInInput}`) await trackProgress( `Previously successful (skipped): ${skippedPreviouslyProcessed}` @@ -1507,8 +2056,19 @@ async function main(trackProgress) { await trackProgress( ` - Skipped (no stripe_customer_id): ${skippedNoStripeIdCount}` ) + await trackProgress(` - Tax info pending: ${taxInfoPendingCount}`) await trackProgress(` - Errors: ${errorCount}`) await trackProgress('') + await trackProgress('Custom fields summary (Recurly -> Stripe metadata):') + for (const fieldName of RECURLY_CUSTOM_FIELD_NAMES) { + await trackProgress( + ` - ${fieldName}: ${customFieldStats[fieldName] || 0}` + ) + } + await trackProgress( + ` - No custom fields: ${customFieldStats.noCustomFields}` + ) + await trackProgress('') if (commit) { await trackProgress( `Success file: ${successOutputPath} (${totalSuccessful} records)` @@ -1524,6 +2084,9 @@ async function main(trackProgress) { await trackProgress( `Errors file: ${errorsOutputPath} (${errorCount} records)` ) + await trackProgress( + `API calls - Recurly: ${finalRateLimiterStats.recurly.totalRequests}, Stripe: ${finalRateLimiterStats.stripe.totalRequests}` + ) if (!commit && dryRunCount > 0) { await trackProgress('')