Verso/services/web/app/src/Features/Project/ProjectCreationHandler.mjs

import OError from '@overleaf/o-error'
import metrics from '@overleaf/metrics'
import logger from '@overleaf/logger'
import Settings from '@overleaf/settings'
import mongodb from 'mongodb-legacy'
import Features from '../../infrastructure/Features.mjs'
import { Project } from '../../models/Project.mjs'
import { Folder } from '../../models/Folder.mjs'
import ProjectEntityUpdateHandler from './ProjectEntityUpdateHandler.mjs'
import ProjectDetailsHandler from './ProjectDetailsHandler.mjs'
import HistoryManager from '../History/HistoryManager.mjs'
import { User } from '../../models/User.mjs'
import fs from 'node:fs'
import path from 'node:path'
import { callbackify } from 'node:util'
import _ from 'lodash'
import AnalyticsManager from '../Analytics/AnalyticsManager.mjs'
import TpdsUpdateSender from '../ThirdPartyDataStore/TpdsUpdateSender.mjs'
import SplitTestHandler from '../SplitTests/SplitTestHandler.mjs'
import ClsiCacheManager from '../Compile/ClsiCacheManager.mjs'
import crypto from 'node:crypto'

const { ObjectId } = mongodb

const MONTH_NAMES = [
  'January',
  'February',
  'March',
  'April',
  'May',
  'June',
  'July',
  'August',
  'September',
  'October',
  'November',
  'December',
]

const templateProjectDir = Features.hasFeature('saas')
  ? 'example-project'
  : 'example-project-sp'

async function createBlankProject(
  ownerId,
  projectName,
  attributes = {},
  options
) {
  const isImport = attributes && attributes.overleaf
  const project = await _createBlankProject(
    ownerId,
    projectName,
    attributes,
    options
  )
  const segmentation = _.pick(attributes, [
    'fromV1TemplateId',
    'fromV1TemplateVersionId',
  ])
  Object.assign(segmentation, attributes.segmentation)
  segmentation.projectId = project._id
  if (isImport) {
    AnalyticsManager.recordEventForUserInBackground(
      ownerId,
      'project-imported',
      segmentation
    )
  } else {
    AnalyticsManager.recordEventForUserInBackground(
      ownerId,
      'project-created',
      segmentation
    )
  }
  return project
}

async function createProjectFromSnippet(ownerId, projectName, docLines) {
  const project = await _createBlankProject(ownerId, projectName)
  AnalyticsManager.recordEventForUserInBackground(ownerId, 'project-created', {
    projectId: project._id,
  })
  await _createRootDoc(project, ownerId, docLines)
  return project
}

// Per-flavour blank-project template, root document name and stored compiler.
function _flavourConfig(flavour) {
  switch (flavour) {
    case 'latex':
      return {
        templateName: 'mainbasic.tex',
        rootDocName: 'main.tex',
        compiler: 'pdflatex',
      }
    case 'typst':
      return {
        templateName: 'mainbasic.typ',
        rootDocName: 'main.typ',
        compiler: 'typst',
      }
    default:
      return {
        templateName: 'mainbasic.qmd',
        rootDocName: 'main.qmd',
        compiler: 'quarto',
      }
  }
}

async function createBasicProject(ownerId, projectName, flavour = 'quarto') {
  // Verso compiles .qmd with Quarto, .tex with latexmk and .typ with Typst;
  // the root file's extension selects the runner (see CompileManager in CLSI).
  // Each flavour is a choice of template + root name + the compiler stored on
  // the project (so the compiler dropdown reflects the engine and LatexRunner
  // never receives a non-LaTeX compiler for a .tex project).
  const { templateName, rootDocName, compiler } = _flavourConfig(flavour)
  const project = await _createBlankProject(ownerId, projectName, { compiler })

  const docLines = await _buildTemplate(templateName, ownerId, projectName)
  await _createRootDoc(project, ownerId, docLines, rootDocName)

  AnalyticsManager.recordEventForUserInBackground(ownerId, 'project-created', {
    projectId: project._id,
  })

  return project
}

function hashProjectContent({ fileEntries, docEntries }) {
  const entries = []
  for (const {
    path,
    file: { hash },
  } of fileEntries) {
    entries.push(`${path}:${hash}`)
  }
  for (const { path, docLines } of docEntries) {
    entries.push(`${path}:${docLines}`)
  }
  entries.sort()
  const hash = crypto.createHash('sha256')
  for (const item of entries) {
    hash.update(item)
  }
  return hash.digest('hex')
}

let exampleProjectContentHash

async function populateClsiCacheForExampleProject(
  ownerId,
  project,
  fileEntries,
  docEntries
) {
  const hash = hashProjectContent({ fileEntries, docEntries })
  if (exampleProjectContentHash && exampleProjectContentHash !== hash) {
    // We need a stable identifier for example projects. Otherwise we will
    // generate a lot of cruft in clsi-cache/GCS.
    const err = new Error('example project content is not static')
    logger.error({ err }, err.message)
    return
  }
  exampleProjectContentHash = hash

  const templateVersionId = `example-project-${hash}`
  const { _id: projectId, imageName } = project
  const found = await ClsiCacheManager.prepareClsiCache(projectId, ownerId, {
    templateVersionId,
    imageName: imageName && path.basename(imageName),
  }).catch(err => {
    logger.error(
      { err, templateVersionId, projectId },
      'failed to prepare clsi-cache from example project'
    )
    return undefined
  })
  if (found === false) {
    ClsiCacheManager.createTemplateClsiCache({
      templateVersionId,
      project,
      fileEntries,
      docEntries,
    }).catch(err => {
      logger.error(
        { err, templateVersionId },
        'failed to create example project clsi-cache'
      )
    })
  }
  return projectId
}

async function createExampleProject(
  ownerId,
  projectName,
  attributes = {},
  flavour = 'latex'
) {
  const { compiler } = _flavourConfig(flavour)
  const project = await _createBlankProject(ownerId, projectName, {
    ...attributes,
    compiler,
  })

  let result
  switch (flavour) {
    case 'quarto':
      result = await _addQuartoExampleProjectFiles(ownerId, projectName, project)
      break
    case 'typst':
      result = await _addTypstExampleProjectFiles(ownerId, projectName, project)
      break
    default:
      result = await _addExampleProjectFiles(ownerId, projectName, project)
  }
  const { fileEntries, docEntries } = result

  if (flavour === 'latex') {
    // clsi-cache warming keys on a single static example; only do it for the
    // long-standing LaTeX example to avoid the "content is not static" guard.
    await populateClsiCacheForExampleProject(
      ownerId,
      project,
      fileEntries,
      docEntries
    )
  }

  AnalyticsManager.recordEventForUserInBackground(ownerId, 'project-created', {
    projectId: project._id,
  })

  return project
}

async function _addExampleProjectFiles(ownerId, projectName, project) {
  const mainDocLines = await _buildTemplate(
    `${templateProjectDir}/main.tex`,
    ownerId,
    projectName
  )
  const rootDoc = await _createRootDoc(
    project,
    ownerId,
    mainDocLines,
    'main.tex'
  )

  const bibDocLines = await _buildTemplate(
    `${templateProjectDir}/sample.bib`,
    ownerId,
    projectName
  )
  const bibDoc = await ProjectEntityUpdateHandler.promises.addDoc(
    project._id,
    project.rootFolder[0]._id,
    'sample.bib',
    bibDocLines,
    ownerId,
    null
  )

  const frogPath = path.join(
    import.meta.dirname,
    `/../../../templates/project_files/${templateProjectDir}/frog.jpg`
  )
  const { fileRef } = await ProjectEntityUpdateHandler.promises.addFile(
    project._id,
    project.rootFolder[0]._id,
    'frog.jpg',
    frogPath,
    null,
    ownerId,
    null
  )
  return {
    fileEntries: [{ path: fileRef.name, file: fileRef }],
    docEntries: [
      { path: 'main.tex', doc: rootDoc, docLines: mainDocLines.join('\n') },
      { path: 'sample.bib', doc: bibDoc, docLines: bibDocLines.join('\n') },
    ],
  }
}

async function _addQuartoExampleProjectFiles(ownerId, projectName, project) {
  const mainDocLines = await _buildTemplate(
    'example-project-quarto/main.qmd',
    ownerId,
    projectName
  )
  const rootDoc = await _createRootDoc(
    project,
    ownerId,
    mainDocLines,
    'main.qmd'
  )

  const imagePath = path.join(
    import.meta.dirname,
    '/../../../templates/project_files/example-project-quarto/frog.jpg'
  )
  const { fileRef } = await ProjectEntityUpdateHandler.promises.addFile(
    project._id,
    project.rootFolder[0]._id,
    'frog.jpg',
    imagePath,
    null,
    ownerId,
    null
  )
  return {
    fileEntries: [{ path: fileRef.name, file: fileRef }],
    docEntries: [
      { path: 'main.qmd', doc: rootDoc, docLines: mainDocLines.join('\n') },
    ],
  }
}

async function _addTypstExampleProjectFiles(ownerId, projectName, project) {
  const mainDocLines = await _buildTemplate(
    'example-project-typst/main.typ',
    ownerId,
    projectName
  )
  const rootDoc = await _createRootDoc(
    project,
    ownerId,
    mainDocLines,
    'main.typ'
  )

  const imagePath = path.join(
    import.meta.dirname,
    '/../../../templates/project_files/example-project-typst/frog.jpg'
  )
  const { fileRef } = await ProjectEntityUpdateHandler.promises.addFile(
    project._id,
    project.rootFolder[0]._id,
    'frog.jpg',
    imagePath,
    null,
    ownerId,
    null
  )
  return {
    fileEntries: [{ path: fileRef.name, file: fileRef }],
    docEntries: [
      { path: 'main.typ', doc: rootDoc, docLines: mainDocLines.join('\n') },
    ],
  }
}

async function _createBlankProject(
  ownerId,
  projectName,
  attributes = {},
  { skipCreatingInTPDS = false } = {}
) {
  metrics.inc('project-creation')
  const timer = new metrics.Timer('project-creation')
  await ProjectDetailsHandler.promises.validateProjectName(projectName)

  const rootFolder = new Folder({ name: 'rootFolder' })

  attributes.lastUpdatedBy = attributes.owner_ref = new ObjectId(ownerId)
  attributes.name = projectName
  const project = new Project(attributes)

  // Initialise the history unless the caller has overridden it in the attributes
  // (to allow scripted creation of projects without full project history)
  if (project.overleaf.history.id == null && !attributes.overleaf) {
    const historyId = await HistoryManager.promises.initializeProject(
      project._id
    )
    if (historyId != null) {
      project.overleaf.history.id = historyId
    }
  }

  // All the projects are initialised with Full Project History. This property
  // is still set for backwards compatibility: Server Pro requires all projects
  // have it set to `true` since SP 4.0
  project.overleaf.history.display = true

  if (Settings.currentImageName) {
    // avoid clobbering any imageName already set in attributes (e.g. importedImageName)
    if (!project.imageName) {
      project.imageName = Settings.currentImageName
    }
  }
  project.rootFolder[0] = rootFolder
  const user = await User.findById(ownerId, {
    'ace.spellCheckLanguage': 1,
    _id: 1,
  })
  project.spellCheckLanguage = user.ace.spellCheckLanguage
  const historyRangesSupportAssignment =
    await SplitTestHandler.promises.getAssignmentForUser(
      user._id,
      'history-ranges-support'
    )
  if (historyRangesSupportAssignment.variant === 'enabled') {
    project.overleaf.history.rangesSupportEnabled = true
  }

  if (attributes.isDebugCopyOf) {
    project.overleaf.isDebugCopyOf = new ObjectId(attributes.isDebugCopyOf)
  }

  await project.save()
  if (!skipCreatingInTPDS) {
    await TpdsUpdateSender.promises.createProject({
      projectId: project._id,
      projectName,
      ownerId,
      userId: ownerId,
    })
  }
  timer.done()
  return project
}

async function _createRootDoc(
  project,
  ownerId,
  docLines,
  rootDocName = 'main.qmd'
) {
  try {
    const { doc } = await ProjectEntityUpdateHandler.promises.addDoc(
      project._id,
      project.rootFolder[0]._id,
      rootDocName,
      docLines,
      ownerId,
      null
    )
    await ProjectEntityUpdateHandler.promises.setRootDoc(project._id, doc._id)
    // update the rootDoc id on the project in memory
    // used to identify the rootResourcePath when doing an initial compile from history
    project.rootDoc_id = doc._id
    return doc
  } catch (error) {
    throw OError.tag(error, 'error adding root doc when creating project')
  }
}

async function _buildTemplate(templateName, userId, projectName) {
  const user = await User.findById(userId, 'first_name last_name')

  const templatePath = path.join(
    import.meta.dirname,
    `/../../../templates/project_files/${templateName}`
  )
  const template = fs.readFileSync(templatePath)
  const data = {
    project_name: projectName,
    user,
    year: new Date().getUTCFullYear(),
    month: MONTH_NAMES[new Date().getUTCMonth()],
  }
  const output = _.template(template.toString())(data)
  return output.split('\n')
}

export default {
  createBlankProject: callbackify(createBlankProject),
  createProjectFromSnippet: callbackify(createProjectFromSnippet),
  createBasicProject: callbackify(createBasicProject),
  createExampleProject: callbackify(createExampleProject),
  promises: {
    createBlankProject,
    createProjectFromSnippet,
    createBasicProject,
    createExampleProject,
  },
}