Merge pull request #13760 from overleaf/em-fetch-utils-web

Use fetch-utils in web

GitOrigin-RevId: cbd0298200bbe42567c6e94934bfb5114fa9b66f

services/web/scripts/delete_orphaned_project_archives.js

@@ -1,12 +1,11 @@
 const Settings = require('@overleaf/settings')
-const OError = require('@overleaf/o-error')
+const { fetchJson } = require('@overleaf/fetch-utils')
 const { waitForDb } = require('../app/src/infrastructure/mongodb')
 const { promiseMapWithLimit } = require('../app/src/util/promises')
 const { getHardDeletedProjectIds } = require('./delete_orphaned_data_helper')
 const TpdsUpdateSender = require('../app/src/Features/ThirdPartyDataStore/TpdsUpdateSender')
 const { promisify } = require('util')
 const { ObjectId } = require('mongodb')
-const fetch = require('node-fetch')
 const sleep = promisify(setTimeout)
 
 const START_OFFSET = process.env.START_OFFSET
@@ -34,15 +33,7 @@ async function main() {
     const url = new URL(`${Settings.apis.project_archiver.url}/project/list`)
     url.searchParams.append('pageToken', pageToken)
     url.searchParams.append('startOffset', startOffset)
-    const response = await fetch(url, {
-      headers: { Accept: 'application/json' },
-    })
-    if (!response.ok) {
-      throw new OError('Failed to get list of projects from project archiver', {
-        status: response.status,
-      })
-    }
-    const { nextPageToken, entries } = await response.json()
+    const { nextPageToken, entries } = await fetchJson(url)
     pageToken = nextPageToken
     startOffset = undefined
 
@@ -60,6 +51,7 @@ async function main() {
     )
   }
 }
+
 async function processBatch(entries) {
   const projectIdToPrefix = new Map()
   for (const { prefix, projectId } of entries) {

services/web/scripts/learn/checkSanitize/scrape.js

@@ -1,7 +1,11 @@
 const Path = require('path')
 const fs = require('fs')
 
-const fetch = require('node-fetch')
+const {
+  fetchString,
+  fetchJson,
+  RequestFailedError,
+} = require('@overleaf/fetch-utils')
 
 const CACHE_IN = Path.join(
   Path.dirname(Path.dirname(Path.dirname(__dirname))),
@@ -17,11 +21,16 @@ async function scrape(baseUrl, page) {
     format: 'json',
     redirects: true,
   }).toString()
-  const response = await fetch(uri)
-  if (response.status !== 200) {
-    console.error(response.status, page, response)
+
+  try {
+    return await fetchString(uri)
+  } catch (err) {
+    if (err instanceof RequestFailedError) {
+      console.error(err.response.status, page, err.response)
+    } else {
+      console.error(err)
+    }
   }
-  return await response.text()
 }
 
 const crypto = require('crypto')
@@ -70,11 +79,18 @@ async function getAllPagesFrom(baseUrl, continueFrom) {
     gaplimit: 100,
     ...continueFrom,
   }).toString()
-  const response = await fetch(uri)
-  if (response.status !== 200) {
-    console.error(response.status, continueFrom, response)
+
+  let blob
+  try {
+    blob = await fetchJson(uri)
+  } catch (err) {
+    if (err instanceof RequestFailedError) {
+      console.error(err.response.status, continueFrom, err.response)
+    } else {
+      console.error(err)
+      throw err
+    }
   }
-  const blob = await response.json()
   const nextContinueFrom = blob && blob.continue
   const pagesRaw = (blob && blob.query && blob.query.pages) || {}
   const pages = Object.values(pagesRaw).map(page => page.title)