Merge pull request #24468 from overleaf/mj-client-side-references

[web] Perform ARS on client-side

GitOrigin-RevId: 19703c82758cae450fe52463ad9612d3a2383ba0
This commit is contained in:
Mathias Jakobsen
2025-09-03 08:05:41 +00:00
committed by Copybot
parent f6820ed794
commit ed0c4c447e
21 changed files with 3040 additions and 26 deletions
@@ -0,0 +1,104 @@
import BasicReferenceIndex from '@/features/ide-react/references/basic-reference-index'
import { expect } from 'chai'
const entry1 = `@article{sample2023,
author = {John Doe},
title = {Sample Title},
journal = {Sample Journal},
year = {2023},
date = {2023-01-01}
}`
const entry2 = `@book{example2022,
author = {Jane Smith},
title = {Example Book},
journal = {Example Journal},
year = {2022}
date = {2022-05-15}
}`
const entry3 = `@inproceedings{test2021,
author = {Alice Johnson},
title = {Test Conference Paper},
booktitle = {Test Conference},
year = {2021},
date = {2021-10-10}
}
`
const fileWithMultipleEntries = `${entry1}\n${entry2}`
const addEntry1 = { path: 'file1.bib', content: entry1 }
const addEntry2 = { path: 'file2.bib', content: entry2 }
const addEntry3 = { path: 'file3.bib', content: entry3 }
const addFileWithMultipleEntries = {
path: 'file5.bib',
content: fileWithMultipleEntries,
}
const deleteEntry2 = 'file2.bib'
describe('BasicReferenceIndex', function () {
beforeEach(function () {
this.index = new BasicReferenceIndex()
})
it('starts with an empty index', function () {
expect(this.index.fileIndex.size).to.equal(0)
expect(this.index.keys.size).to.equal(0)
})
describe('updateIndex', function () {
it('Adds entry to index and keys', function () {
const changes = { updates: [addEntry1], deletes: [] }
const keys = this.index.updateIndex(changes)
expect(this.index.fileIndex.size).to.equal(1)
expect(this.index.fileIndex.get('file1.bib')).to.deep.equal(
new Set(['sample2023'])
)
expect(keys).to.deep.equal(new Set(['sample2023']))
})
it("doesn't forget existing keys when adding new entries", function () {
const changes = { updates: [addEntry1, addEntry2], deletes: [] }
const keys = this.index.updateIndex(changes)
expect(this.index.fileIndex.size).to.equal(2)
expect(keys).to.deep.equal(new Set(['sample2023', 'example2022']))
const additionalChanges = { updates: [addEntry3], deletes: [] }
const updatedKeys = this.index.updateIndex(additionalChanges)
expect(this.index.fileIndex.size).to.equal(3)
expect(updatedKeys).to.deep.equal(
new Set(['sample2023', 'example2022', 'test2021'])
)
})
it('removes keys when files are deleted', function () {
const changes = {
updates: [addEntry1, addEntry2, addEntry3],
deletes: [],
}
this.index.updateIndex(changes)
expect(this.index.fileIndex.size).to.equal(3)
expect(this.index.keys).to.deep.equal(
new Set(['sample2023', 'example2022', 'test2021'])
)
const deletionChanges = { updates: [], deletes: [deleteEntry2] }
const keysAfterDeletion = this.index.updateIndex(deletionChanges)
expect(this.index.fileIndex.size).to.equal(2)
expect(keysAfterDeletion).to.deep.equal(
new Set(['sample2023', 'test2021'])
)
})
it('handles multiple entries in a single file', function () {
const changes = { updates: [addFileWithMultipleEntries], deletes: [] }
const keys = this.index.updateIndex(changes)
expect(this.index.fileIndex.size).to.equal(1)
expect(this.index.fileIndex.get('file5.bib')).to.deep.equal(
new Set(['sample2023', 'example2022'])
)
expect(keys).to.deep.equal(new Set(['sample2023', 'example2022']))
})
})
})
@@ -0,0 +1,83 @@
import { expect } from 'chai'
import { ReferenceIndex } from '@/features/ide-react/references/reference-index'
class TestedReferenceIndex extends ReferenceIndex {
updateIndex(): void {
throw new Error('This is a test implementation')
}
}
describe('ReferenceIndex', function () {
beforeEach(function () {
this.index = new TestedReferenceIndex()
})
describe('parseEntries', function () {
it('should parse bib entry', function () {
const content = `
@article{sample2023,
author = {John Doe},
title = {Sample Title},
journal = {Sample Journal},
year = {2023},
date = {2023-01-01}
}
`
const entries = this.index.parseEntries(content)
expect(entries).to.have.lengthOf(1)
expect(entries[0]).to.deep.equal({
EntryKey: 'sample2023',
EntryType: 'article',
Fields: {
author: 'John Doe',
title: 'Sample Title',
journal: 'Sample Journal',
year: '2023',
date: '2023-01-01',
},
ObjectType: 'entry',
})
})
it('should default missing fields to empty strings', function () {
const content = `@article{sample2023,
author = {John Doe},
title = {Sample Title}
}`
const entries = this.index.parseEntries(content)
expect(entries).to.have.lengthOf(1)
expect(entries[0]).to.deep.equal({
EntryKey: 'sample2023',
EntryType: 'article',
Fields: {
author: 'John Doe',
title: 'Sample Title',
journal: '',
year: '',
date: '',
},
ObjectType: 'entry',
})
})
it('should handle multiple entries', function () {
const content = `@article{sample2023,
author = {John Doe},
title = {Sample Title},
journal = {Sample Journal},
year = {2023},
date = {2023-01-01}
}
@book{example2022,
author = {Jane Smith},
title = {Example Book},
journal = {Example Journal},
year = {2022},
date = {2022-05-15}
}`
const entries = this.index.parseEntries(content)
expect(entries).to.have.lengthOf(2)
expect(entries[0].EntryKey).to.equal('sample2023')
expect(entries[1].EntryKey).to.equal('example2022')
})
})
})
@@ -0,0 +1,326 @@
import { ReferenceIndexer } from '@/features/ide-react/references/reference-indexer'
import { generateMD5Hash } from '@/shared/utils/md5'
import sinon from 'sinon'
const entry1 = `@article{sample2023,
author = {John Doe},
title = {Sample Title},
journal = {Sample Journal},
year = {2023},
date = {2023-01-01}
}`
const entry2 = `@book{example2022,
author = {Jane Smith},
title = {Example Book},
journal = {Example Journal},
year = {2022}
date = {2022-05-15}
}`
const entry3 = `@article{sample2024,
author = {John Doe},
title = {Sample Title},
journal = {Sample Journal},
year = {2024},
date = {2024-01-01}
}`
const entry4 = `@book{example2025,
author = {Jane Smith},
title = {Example Book},
journal = {Example Journal},
year = {2025}
date = {2025-05-15}
}`
const snapshotWithData = ({
docs,
files,
}: {
docs?: Record<string, string>
files?: Record<string, string>
}) => {
return {
getDocPaths: sinon.spy(() => Object.keys(docs ?? {})),
getDocContents: sinon.spy(path => (docs ? (docs[path] ?? null) : null)),
getBinaryFilePathsWithHash: sinon.spy(() => {
return Object.entries(files ?? {}).map(([path, content]) => ({
path,
hash: generateMD5Hash(content),
size: content.length,
}))
}),
getBinaryFileContents: sinon.spy(async path =>
files ? (files[path] ?? null) : null
),
}
}
const IGNORED_SIGNAL = new AbortController().signal
describe('ReferenceIndexer', function () {
it('it should index bib docs', async function () {
const referencer = new ReferenceIndexer()
const snapshot = snapshotWithData({
docs: {
'refs.bib': entry1,
'refs2.bib': entry2,
'other.tex': 'Not a bib file',
},
})
const result = await referencer.updateFromSnapshot(snapshot, {
signal: IGNORED_SIGNAL,
})
expect(snapshot.getDocPaths).to.have.been.calledOnce
expect(snapshot.getDocContents).to.have.been.calledTwice
expect(snapshot.getDocContents).to.have.been.calledWith('refs.bib')
expect(snapshot.getDocContents).to.have.been.calledWith('refs2.bib')
expect(snapshot.getDocContents).to.not.have.been.calledWith('other.tex')
expect(snapshot.getBinaryFileContents).to.not.have.been.called
expect(result).to.deep.equal(new Set(['sample2023', 'example2022']))
})
it('it should index bib binary files', async function () {
const referencer = new ReferenceIndexer()
const snapshot = snapshotWithData({
files: {
'refs.bib': entry1,
'refs2.bib': entry2,
'image.png': 'Not a bib file',
},
})
const result = await referencer.updateFromSnapshot(snapshot, {
signal: IGNORED_SIGNAL,
})
expect(snapshot.getDocPaths).to.have.been.calledOnce
expect(snapshot.getDocContents).to.not.have.been.called
expect(snapshot.getBinaryFilePathsWithHash).to.have.been.calledOnce
expect(snapshot.getBinaryFileContents).to.have.been.calledTwice
expect(snapshot.getBinaryFileContents).to.have.been.calledWith('refs.bib')
expect(snapshot.getBinaryFileContents).to.have.been.calledWith('refs2.bib')
expect(snapshot.getBinaryFileContents).to.not.have.been.calledWith(
'image.png'
)
expect(result).to.deep.equal(new Set(['sample2023', 'example2022']))
})
it('it should index both bib docs and binary files', async function () {
const referencer = new ReferenceIndexer()
const snapshot = snapshotWithData({
docs: {
'refs.bib': entry1,
'other.tex': 'Not a bib file',
},
files: {
'refs2.bib': entry2,
'image.png': 'Not a bib file',
},
})
const result = await referencer.updateFromSnapshot(snapshot, {
signal: IGNORED_SIGNAL,
})
expect(snapshot.getDocPaths).to.have.been.calledOnce
expect(snapshot.getDocContents).to.have.been.calledOnce
expect(snapshot.getDocContents).to.have.been.calledWith('refs.bib')
expect(snapshot.getDocContents).to.not.have.been.calledWith('other.tex')
expect(snapshot.getBinaryFilePathsWithHash).to.have.been.calledOnce
expect(snapshot.getBinaryFileContents).to.have.been.calledOnce
expect(snapshot.getBinaryFileContents).to.have.been.calledWith('refs2.bib')
expect(snapshot.getBinaryFileContents).to.not.have.been.calledWith(
'image.png'
)
expect(result).to.deep.equal(new Set(['sample2023', 'example2022']))
})
it('should not fetch binary files if unchanged', async function () {
const referencer = new ReferenceIndexer()
const initialSnapshot = snapshotWithData({
files: {
'refs.bib': entry1,
},
})
const initialResult = await referencer.updateFromSnapshot(initialSnapshot, {
signal: IGNORED_SIGNAL,
})
expect(initialSnapshot.getDocPaths).to.have.been.calledOnce
expect(initialSnapshot.getBinaryFilePathsWithHash).to.have.been.calledOnce
expect(initialSnapshot.getBinaryFileContents).to.have.been.calledOnceWith(
'refs.bib'
)
expect(initialResult).to.deep.equal(new Set(['sample2023']))
// Second snapshot with same files, should not fetch contents again
const secondSnapshot = snapshotWithData({
files: {
'refs.bib': entry1,
},
})
const secondResult = await referencer.updateFromSnapshot(secondSnapshot, {
signal: IGNORED_SIGNAL,
})
expect(secondSnapshot.getDocPaths).to.have.been.calledOnce
expect(secondSnapshot.getBinaryFilePathsWithHash).to.have.been.calledOnce
expect(secondSnapshot.getBinaryFileContents).to.not.have.been.called
expect(secondResult).to.deep.equal(new Set(['sample2023']))
})
it('should fetch changed binary file', async function () {
const referencer = new ReferenceIndexer()
const initialSnapshot = snapshotWithData({
files: {
'refs.bib': entry1,
},
})
const initialResult = await referencer.updateFromSnapshot(initialSnapshot, {
signal: IGNORED_SIGNAL,
})
expect(initialSnapshot.getDocPaths).to.have.been.calledOnce
expect(initialSnapshot.getBinaryFilePathsWithHash).to.have.been.calledOnce
expect(initialSnapshot.getBinaryFileContents).to.have.been.calledOnceWith(
'refs.bib'
)
expect(initialResult).to.deep.equal(new Set(['sample2023']))
// Second snapshot with a different file, should fetch contents again
const secondSnapshot = snapshotWithData({
files: {
'refs.bib': entry2,
},
})
const secondResult = await referencer.updateFromSnapshot(secondSnapshot, {
signal: IGNORED_SIGNAL,
})
expect(secondSnapshot.getDocPaths).to.have.been.calledOnce
expect(secondSnapshot.getBinaryFilePathsWithHash).to.have.been.calledOnce
expect(initialSnapshot.getBinaryFileContents).to.have.been.calledOnceWith(
'refs.bib'
)
expect(secondResult).to.deep.equal(new Set(['example2022']))
})
it('should update changed doc', async function () {
const referencer = new ReferenceIndexer()
const initialSnapshot = snapshotWithData({
docs: {
'refs.bib': entry1,
},
})
const initialResult = await referencer.updateFromSnapshot(initialSnapshot, {
signal: IGNORED_SIGNAL,
})
expect(initialResult).to.deep.equal(new Set(['sample2023']))
const secondSnapshot = snapshotWithData({
docs: {
'refs.bib': entry2,
},
})
const secondResult = await referencer.updateFromSnapshot(secondSnapshot, {
signal: IGNORED_SIGNAL,
})
expect(secondResult).to.deep.equal(new Set(['example2022']))
})
it('should notice deleted files', async function () {
const referencer = new ReferenceIndexer()
const initialSnapshot = snapshotWithData({
files: {
'refs.bib': entry1,
'refs2.bib': entry2,
},
})
const initialResult = await referencer.updateFromSnapshot(initialSnapshot, {
signal: IGNORED_SIGNAL,
})
expect(initialSnapshot.getDocPaths).to.have.been.calledOnce
expect(initialSnapshot.getBinaryFilePathsWithHash).to.have.been.calledOnce
expect(initialSnapshot.getBinaryFileContents).to.have.been.calledTwice
expect(initialResult).to.deep.equal(new Set(['sample2023', 'example2022']))
// Second snapshot with one file removed, should update index
const secondSnapshot = snapshotWithData({
files: {
'refs.bib': entry1,
},
})
const secondResult = await referencer.updateFromSnapshot(secondSnapshot, {
signal: IGNORED_SIGNAL,
})
expect(secondSnapshot.getDocPaths).to.have.been.calledOnce
expect(secondSnapshot.getBinaryFilePathsWithHash).to.have.been.calledOnce
expect(secondSnapshot.getBinaryFileContents).to.not.have.been.called
expect(secondResult).to.deep.equal(new Set(['sample2023']))
})
it('should notice deleted docs', async function () {
const referencer = new ReferenceIndexer()
const initialSnapshot = snapshotWithData({
docs: {
'refs.bib': entry1,
'refs2.bib': entry2,
},
})
const initialResult = await referencer.updateFromSnapshot(initialSnapshot, {
signal: IGNORED_SIGNAL,
})
expect(initialSnapshot.getDocPaths).to.have.been.calledOnce
expect(initialSnapshot.getDocContents).to.have.been.calledTwice
expect(initialResult).to.deep.equal(new Set(['sample2023', 'example2022']))
// Second snapshot with one doc removed, should update index
const secondSnapshot = snapshotWithData({
docs: {
'refs.bib': entry1,
},
})
const secondResult = await referencer.updateFromSnapshot(secondSnapshot, {
signal: IGNORED_SIGNAL,
})
expect(secondSnapshot.getDocPaths).to.have.been.calledOnce
expect(secondSnapshot.getDocContents).to.have.been.calledOnce
expect(secondResult).to.deep.equal(new Set(['sample2023']))
})
it('should abort when signalled', async function () {
const referencer = new ReferenceIndexer()
const snapshot = snapshotWithData({
files: {
'refs.bib': entry1,
'refs2.bib': entry2,
},
})
const controller = new AbortController()
controller.abort()
const result = await referencer.updateFromSnapshot(snapshot, {
signal: controller.signal,
})
expect(result).to.deep.equal(new Set())
})
it('should respect data budget', async function () {
async function testWithDataBudget(budget: number, keys: Set<string>) {
const referencer = new ReferenceIndexer()
const snapshot = snapshotWithData({
docs: {
'a.bib': entry1, // 140 bytes
'b.bib': entry2, // 140 bytes
'c.bib': entry3, // 140 bytes
'd.bib': entry4, // 140 bytes
},
})
const result = await referencer.updateFromSnapshot(snapshot, {
signal: IGNORED_SIGNAL,
dataLimit: budget,
})
expect(result).to.deep.equal(keys)
}
await testWithDataBudget(
1000,
new Set(['sample2023', 'example2022', 'sample2024', 'example2025'])
)
await testWithDataBudget(300, new Set(['sample2023', 'example2022']))
await testWithDataBudget(200, new Set(['sample2023']))
await testWithDataBudget(100, new Set())
})
})
@@ -381,6 +381,11 @@ describe('autocomplete', { scrollBehavior: false }, function () {
value={{
referenceKeys: new Set(['ref-1', 'ref-2', 'ref-3']),
indexAllReferences: cy.stub(),
searchLocalReferences() {
return Promise.resolve({
hits: [],
})
},
}}
>
{children}
@@ -47,6 +47,7 @@ import {
} from '@/shared/context/types/project-metadata'
import { UserId } from '../../../types/user'
import { ProjectCompiler } from '../../../types/project-settings'
import { ReferencesContext } from '@/features/ide-react/context/references-context'
// these constants can be imported in tests instead of
// using magic strings
@@ -243,6 +244,7 @@ export function EditorProviders({
}),
LayoutProvider: makeLayoutProvider(layoutContext),
ProjectProvider: makeProjectProvider(project),
ReferencesProvider: makeReferencesProvider(),
...providers,
}}
>
@@ -251,6 +253,27 @@ export function EditorProviders({
)
}
const makeReferencesProvider = () => {
const ReferencesProvider: FC<PropsWithChildren> = ({ children }) => {
return (
<ReferencesContext.Provider
value={{
referenceKeys: new Set(),
indexAllReferences: () => Promise.resolve(),
searchLocalReferences() {
return Promise.resolve({
hits: [],
})
},
}}
>
{children}
</ReferencesContext.Provider>
)
}
return ReferencesProvider
}
const makeConnectionProvider = (socket: Socket) => {
const ConnectionProvider: FC<PropsWithChildren> = ({ children }) => {
const [value] = useState(() => ({
@@ -37,6 +37,13 @@ describe('ProjectSnapshot', function () {
contents: "We're done here",
hash: 'dddddddddddddddddddddddddddddddddddddddd',
},
'bibliography.bib': {
contents:
'@book{example2020,\n title={An example book},\n author={Doe, John},\n year={2020},\n publisher={Publisher}\n}\n'.repeat(
60_000
), // 6.5MB
hash: 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
},
}
const chunk = {
@@ -68,6 +75,13 @@ describe('ProjectSnapshot', function () {
byteLength: 97080,
},
},
{
pathname: 'bibliography.bib',
file: {
hash: files['bibliography.bib'].hash,
byteLength: files['bibliography.bib'].contents.length,
},
},
],
timestamp: '2025-01-01T12:00:00.000Z',
},
@@ -214,6 +228,45 @@ describe('ProjectSnapshot', function () {
)
})
})
describe('getBinaryFilePathsWithHash()', function () {
it('returns the binary files', function () {
const binaries = snapshot.getBinaryFilePathsWithHash()
expect(binaries).to.deep.equal([
{
path: 'frog.jpg',
hash: 'cccccccccccccccccccccccccccccccccccccccc',
size: 97080,
},
{
path: 'bibliography.bib',
hash: files['bibliography.bib'].hash,
size: files['bibliography.bib'].contents.length,
},
])
})
})
describe('getBinaryFileContents', function () {
beforeEach(function () {
mockBlobs(['bibliography.bib'])
})
it('can fetch whole file', async function () {
const blob = await snapshot.getBinaryFileContents('bibliography.bib')
expect(blob).to.equal(files['bibliography.bib'].contents)
})
// NOTE: fetch-mock does not support the .response.body.pipeThrough API,
// so this test is skipped for now.
// eslint-disable-next-line mocha/no-skipped-tests
it.skip('can fetch part of file', async function () {
const blob = await snapshot.getBinaryFileContents('bibliography.bib', {
maxSize: 100,
})
expect(blob).to.equal(files['bibliography.bib'].contents.slice(0, 100))
})
})
})
describe('concurrency', function () {