import OError from '@overleaf/o-error' import metrics from '@overleaf/metrics' import logger from '@overleaf/logger' import Settings from '@overleaf/settings' import mongodb from 'mongodb-legacy' import Features from '../../infrastructure/Features.mjs' import { Project } from '../../models/Project.mjs' import { Folder } from '../../models/Folder.mjs' import ProjectEntityUpdateHandler from './ProjectEntityUpdateHandler.mjs' import ProjectDetailsHandler from './ProjectDetailsHandler.mjs' import HistoryManager from '../History/HistoryManager.mjs' import { User } from '../../models/User.mjs' import fs from 'node:fs' import path from 'node:path' import { callbackify } from 'node:util' import _ from 'lodash' import AnalyticsManager from '../Analytics/AnalyticsManager.mjs' import TpdsUpdateSender from '../ThirdPartyDataStore/TpdsUpdateSender.mjs' import SplitTestHandler from '../SplitTests/SplitTestHandler.mjs' import ClsiCacheManager from '../Compile/ClsiCacheManager.mjs' import crypto from 'node:crypto' const { ObjectId } = mongodb const MONTH_NAMES = [ 'January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December', ] const templateProjectDir = Features.hasFeature('saas') ? 'example-project' : 'example-project-sp' async function createBlankProject( ownerId, projectName, attributes = {}, options ) { const isImport = attributes && attributes.overleaf const project = await _createBlankProject( ownerId, projectName, attributes, options ) const segmentation = _.pick(attributes, [ 'fromV1TemplateId', 'fromV1TemplateVersionId', ]) Object.assign(segmentation, attributes.segmentation) segmentation.projectId = project._id if (isImport) { AnalyticsManager.recordEventForUserInBackground( ownerId, 'project-imported', segmentation ) } else { AnalyticsManager.recordEventForUserInBackground( ownerId, 'project-created', segmentation ) } return project } async function createProjectFromSnippet(ownerId, projectName, docLines) { const project = await _createBlankProject(ownerId, projectName) AnalyticsManager.recordEventForUserInBackground(ownerId, 'project-created', { projectId: project._id, }) await _createRootDoc(project, ownerId, docLines) return project } // Per-flavour blank-project template, root document name and stored compiler. function _flavourConfig(flavour) { switch (flavour) { case 'latex': return { templateName: 'mainbasic.tex', rootDocName: 'main.tex', compiler: 'pdflatex', } case 'typst': return { templateName: 'mainbasic.typ', rootDocName: 'main.typ', compiler: 'typst', } default: return { templateName: 'mainbasic.qmd', rootDocName: 'main.qmd', compiler: 'quarto', } } } async function createBasicProject(ownerId, projectName, flavour = 'quarto') { // Verso compiles .qmd with Quarto, .tex with latexmk and .typ with Typst; // the root file's extension selects the runner (see CompileManager in CLSI). // Each flavour is a choice of template + root name + the compiler stored on // the project (so the compiler dropdown reflects the engine and LatexRunner // never receives a non-LaTeX compiler for a .tex project). const { templateName, rootDocName, compiler } = _flavourConfig(flavour) const project = await _createBlankProject(ownerId, projectName, { compiler }) const docLines = await _buildTemplate(templateName, ownerId, projectName) await _createRootDoc(project, ownerId, docLines, rootDocName) AnalyticsManager.recordEventForUserInBackground(ownerId, 'project-created', { projectId: project._id, }) return project } function hashProjectContent({ fileEntries, docEntries }) { const entries = [] for (const { path, file: { hash }, } of fileEntries) { entries.push(`${path}:${hash}`) } for (const { path, docLines } of docEntries) { entries.push(`${path}:${docLines}`) } entries.sort() const hash = crypto.createHash('sha256') for (const item of entries) { hash.update(item) } return hash.digest('hex') } let exampleProjectContentHash async function populateClsiCacheForExampleProject( ownerId, project, fileEntries, docEntries ) { const hash = hashProjectContent({ fileEntries, docEntries }) if (exampleProjectContentHash && exampleProjectContentHash !== hash) { // We need a stable identifier for example projects. Otherwise we will // generate a lot of cruft in clsi-cache/GCS. const err = new Error('example project content is not static') logger.error({ err }, err.message) return } exampleProjectContentHash = hash const templateVersionId = `example-project-${hash}` const { _id: projectId, imageName } = project const found = await ClsiCacheManager.prepareClsiCache(projectId, ownerId, { templateVersionId, imageName: imageName && path.basename(imageName), }).catch(err => { logger.error( { err, templateVersionId, projectId }, 'failed to prepare clsi-cache from example project' ) return undefined }) if (found === false) { ClsiCacheManager.createTemplateClsiCache({ templateVersionId, project, fileEntries, docEntries, }).catch(err => { logger.error( { err, templateVersionId }, 'failed to create example project clsi-cache' ) }) } return projectId } async function createExampleProject( ownerId, projectName, attributes = {}, flavour = 'latex' ) { const { compiler } = _flavourConfig(flavour) const project = await _createBlankProject(ownerId, projectName, { ...attributes, compiler, }) let result switch (flavour) { case 'quarto': result = await _addQuartoExampleProjectFiles(ownerId, projectName, project) break case 'typst': result = await _addTypstExampleProjectFiles(ownerId, projectName, project) break default: result = await _addExampleProjectFiles(ownerId, projectName, project) } const { fileEntries, docEntries } = result if (flavour === 'latex') { // clsi-cache warming keys on a single static example; only do it for the // long-standing LaTeX example to avoid the "content is not static" guard. await populateClsiCacheForExampleProject( ownerId, project, fileEntries, docEntries ) } AnalyticsManager.recordEventForUserInBackground(ownerId, 'project-created', { projectId: project._id, }) return project } async function _addExampleProjectFiles(ownerId, projectName, project) { const mainDocLines = await _buildTemplate( `${templateProjectDir}/main.tex`, ownerId, projectName ) const rootDoc = await _createRootDoc( project, ownerId, mainDocLines, 'main.tex' ) const bibDocLines = await _buildTemplate( `${templateProjectDir}/sample.bib`, ownerId, projectName ) const bibDoc = await ProjectEntityUpdateHandler.promises.addDoc( project._id, project.rootFolder[0]._id, 'sample.bib', bibDocLines, ownerId, null ) const frogPath = path.join( import.meta.dirname, `/../../../templates/project_files/${templateProjectDir}/frog.jpg` ) const { fileRef } = await ProjectEntityUpdateHandler.promises.addFile( project._id, project.rootFolder[0]._id, 'frog.jpg', frogPath, null, ownerId, null ) return { fileEntries: [{ path: fileRef.name, file: fileRef }], docEntries: [ { path: 'main.tex', doc: rootDoc, docLines: mainDocLines.join('\n') }, { path: 'sample.bib', doc: bibDoc, docLines: bibDocLines.join('\n') }, ], } } async function _addQuartoExampleProjectFiles(ownerId, projectName, project) { const mainDocLines = await _buildTemplate( 'example-project-quarto/main.qmd', ownerId, projectName ) const rootDoc = await _createRootDoc( project, ownerId, mainDocLines, 'main.qmd' ) const imagePath = path.join( import.meta.dirname, '/../../../templates/project_files/example-project-quarto/frog.jpg' ) const { fileRef } = await ProjectEntityUpdateHandler.promises.addFile( project._id, project.rootFolder[0]._id, 'frog.jpg', imagePath, null, ownerId, null ) return { fileEntries: [{ path: fileRef.name, file: fileRef }], docEntries: [ { path: 'main.qmd', doc: rootDoc, docLines: mainDocLines.join('\n') }, ], } } async function _addTypstExampleProjectFiles(ownerId, projectName, project) { const mainDocLines = await _buildTemplate( 'example-project-typst/main.typ', ownerId, projectName ) const rootDoc = await _createRootDoc( project, ownerId, mainDocLines, 'main.typ' ) const imagePath = path.join( import.meta.dirname, '/../../../templates/project_files/example-project-typst/frog.jpg' ) const { fileRef } = await ProjectEntityUpdateHandler.promises.addFile( project._id, project.rootFolder[0]._id, 'frog.jpg', imagePath, null, ownerId, null ) return { fileEntries: [{ path: fileRef.name, file: fileRef }], docEntries: [ { path: 'main.typ', doc: rootDoc, docLines: mainDocLines.join('\n') }, ], } } async function _createBlankProject( ownerId, projectName, attributes = {}, { skipCreatingInTPDS = false } = {} ) { metrics.inc('project-creation') const timer = new metrics.Timer('project-creation') await ProjectDetailsHandler.promises.validateProjectName(projectName) const rootFolder = new Folder({ name: 'rootFolder' }) attributes.lastUpdatedBy = attributes.owner_ref = new ObjectId(ownerId) attributes.name = projectName const project = new Project(attributes) // Initialise the history unless the caller has overridden it in the attributes // (to allow scripted creation of projects without full project history) if (project.overleaf.history.id == null && !attributes.overleaf) { const historyId = await HistoryManager.promises.initializeProject( project._id ) if (historyId != null) { project.overleaf.history.id = historyId } } // All the projects are initialised with Full Project History. This property // is still set for backwards compatibility: Server Pro requires all projects // have it set to `true` since SP 4.0 project.overleaf.history.display = true if (Settings.currentImageName) { // avoid clobbering any imageName already set in attributes (e.g. importedImageName) if (!project.imageName) { project.imageName = Settings.currentImageName } } project.rootFolder[0] = rootFolder const user = await User.findById(ownerId, { 'ace.spellCheckLanguage': 1, _id: 1, }) project.spellCheckLanguage = user.ace.spellCheckLanguage const historyRangesSupportAssignment = await SplitTestHandler.promises.getAssignmentForUser( user._id, 'history-ranges-support' ) if (historyRangesSupportAssignment.variant === 'enabled') { project.overleaf.history.rangesSupportEnabled = true } if (attributes.isDebugCopyOf) { project.overleaf.isDebugCopyOf = new ObjectId(attributes.isDebugCopyOf) } await project.save() if (!skipCreatingInTPDS) { await TpdsUpdateSender.promises.createProject({ projectId: project._id, projectName, ownerId, userId: ownerId, }) } timer.done() return project } async function _createRootDoc( project, ownerId, docLines, rootDocName = 'main.qmd' ) { try { const { doc } = await ProjectEntityUpdateHandler.promises.addDoc( project._id, project.rootFolder[0]._id, rootDocName, docLines, ownerId, null ) await ProjectEntityUpdateHandler.promises.setRootDoc(project._id, doc._id) // update the rootDoc id on the project in memory // used to identify the rootResourcePath when doing an initial compile from history project.rootDoc_id = doc._id return doc } catch (error) { throw OError.tag(error, 'error adding root doc when creating project') } } async function _buildTemplate(templateName, userId, projectName) { const user = await User.findById(userId, 'first_name last_name') const templatePath = path.join( import.meta.dirname, `/../../../templates/project_files/${templateName}` ) const template = fs.readFileSync(templatePath) const data = { project_name: projectName, user, year: new Date().getUTCFullYear(), month: MONTH_NAMES[new Date().getUTCMonth()], } const output = _.template(template.toString())(data) return output.split('\n') } export default { createBlankProject: callbackify(createBlankProject), createProjectFromSnippet: callbackify(createProjectFromSnippet), createBasicProject: callbackify(createBasicProject), createExampleProject: callbackify(createExampleProject), promises: { createBlankProject, createProjectFromSnippet, createBasicProject, createExampleProject, }, }