From a5f9e6ecd65b93785f3034e1a3d72d545acb3fa1 Mon Sep 17 00:00:00 2001 From: Simon Gardner Date: Fri, 6 Feb 2026 17:17:37 +0000 Subject: [PATCH] web: fix: BOM parsing and rate limiting bugs in Recurly -> Stripe customer upsert script (#31398) * fix: add check for BOM to csv parser * fix bug in throttling that allowed 'stampeding herds' * fix theoretical error handling bug in throttle() GitOrigin-RevId: cfe5da63281590b03ababef370a2fb1a316f2dbf --- .../migrate_recurly_customers_to_stripe.mjs | 69 +++++++++++-------- 1 file changed, 42 insertions(+), 27 deletions(-) diff --git a/services/web/scripts/recurly/migrate_recurly_customers_to_stripe.mjs b/services/web/scripts/recurly/migrate_recurly_customers_to_stripe.mjs index e8483339da..c1ce2d7ff0 100644 --- a/services/web/scripts/recurly/migrate_recurly_customers_to_stripe.mjs +++ b/services/web/scripts/recurly/migrate_recurly_customers_to_stripe.mjs @@ -553,42 +553,56 @@ class RateLimiter { this.windowMs = windowMs this.requests = [] // timestamps of recent requests this.totalRequests = 0 + this._pending = Promise.resolve() } /** * Wait if necessary to stay within rate limits, then record the request. */ async throttle() { - const now = Date.now() + this._pending = this._pending + .catch(error => { + // this should never happen since setTimeout or logDebug are very unlikely to ever fail + // but if it does, we log it and continue without blocking the queue (fail-open) + logWarn(`Rate limiter chain error for ${this.name}`, { + error: error?.message || String(error), + }) + }) + .then(async () => { + while (true) { + const now = Date.now() - // Remove requests outside the window - const windowStart = now - this.windowMs - this.requests = this.requests.filter(ts => ts > windowStart) + // Remove requests outside the window + const windowStart = now - this.windowMs + this.requests = this.requests.filter(ts => ts > windowStart) - // If at limit, wait until the oldest request exits the window - if (this.requests.length >= this.maxRequests) { - const oldestRequest = this.requests[0] - const waitTime = oldestRequest - windowStart + 1 - if (waitTime > 0) { - logDebug( - `Rate limit throttle for ${this.name}`, - { - waitMs: waitTime, - currentRequests: this.requests.length, - maxRequests: this.maxRequests, - }, - { verboseOnly: true } - ) - await setTimeout(waitTime) - } - // Clean up again after waiting - const newNow = Date.now() - this.requests = this.requests.filter(ts => ts > newNow - this.windowMs) - } + // If at limit, wait until the oldest request exits the window + if (this.requests.length >= this.maxRequests) { + const oldestRequest = this.requests[0] + const waitTime = oldestRequest - windowStart + 1 + if (waitTime > 0) { + logDebug( + `Rate limit throttle for ${this.name}`, + { + waitMs: waitTime, + currentRequests: this.requests.length, + maxRequests: this.maxRequests, + }, + { verboseOnly: true } + ) + await setTimeout(waitTime) + continue + } + } - // Record this request - this.requests.push(Date.now()) - this.totalRequests++ + // Record this request + this.requests.push(Date.now()) + this.totalRequests++ + break + } + }) + + return this._pending } /** @@ -1851,6 +1865,7 @@ async function main(trackProgress) { const parser = csv.parse({ columns: true, trim: true, + bom: true, skip_empty_lines: true, relax_column_count: true, relax_column_count_less: true,