[history-ot] initial implementation of using doc-level history-ot (#25054)

* [history-v1-ot] initial implementation of using doc-level history-v1-ot * [web] fix advancing of the otMigrationStage Use 'nextStage' for the user provided, desired stage when advancing. Co-authored-by: Brian Gough <brian.gough@overleaf.com> * [document-updater] document size check in editor-core * [history-ot] rename history-v1-ot to history-ot and add types * [history-ot] apply review feedback - remove extra !! - merge variable assignment when processing diff-match-match output - add helper function for getting docstore lines view of StringFileData Co-authored-by: Alf Eaton <alf.eaton@overleaf.com> * Revert "[document-updater] add safe rollback point for history-ot (#25283)" This reverts commit d7230dd14a379a27d2c6ab03a006463a18979d06 Signed-off-by: Jakob Ackermann <jakob.ackermann@overleaf.com> --------- Signed-off-by: Jakob Ackermann <jakob.ackermann@overleaf.com> Co-authored-by: Brian Gough <brian.gough@overleaf.com> Co-authored-by: Alf Eaton <alf.eaton@overleaf.com> GitOrigin-RevId: 89c497782adb0427635d50d02263d6f535b12481
2025-05-08 08:05:44 +00:00
parent 4d93187e58
commit e8b5ee2ff9
48 changed files with 1828 additions and 223 deletions
@@ -2,7 +2,7 @@
 // Migrated from services/web/frontend/js/ide/editor/Document.js

 import RangesTracker from '@overleaf/ranges-tracker'
-import { ShareJsDoc } from './share-js-doc'
+import { OTType, ShareJsDoc } from './share-js-doc'
 import { debugConsole } from '@/utils/debugging'
 import { Socket } from '@/features/ide-react/connection/types/socket'
 import { IdeEventEmitter } from '@/features/ide-react/create-ide-event-emitter'
@@ -28,6 +28,7 @@ import {
 } from '@/features/ide-react/editor/types/document'
 import { ThreadId } from '../../../../../types/review-panel/review-panel'
 import getMeta from '@/utils/meta'
+import OError from '@overleaf/o-error'

 const MAX_PENDING_OP_SIZE = 64

@@ -447,16 +448,36 @@ export class DocumentContainer extends EventEmitter {
        'joinDoc',
        this.doc_id,
        this.doc.getVersion(),
-        { encodeRanges: true, age: this.doc.getTimeSinceLastServerActivity() },
-        (error, docLines, version, updates, ranges) => {
+        {
+          encodeRanges: true,
+          age: this.doc.getTimeSinceLastServerActivity(),
+          supportsHistoryV1OT: true,
+        },
+        (
+          error,
+          docLines,
+          version,
+          updates,
+          ranges,
+          type = 'sharejs-text-ot'
+        ) => {
          if (error) {
            callback?.(error)
            return
          }
          this.joined = true
          this.doc?.catchUp(updates)
-          this.decodeRanges(ranges)
-          this.catchUpRanges(ranges?.changes, ranges?.comments)
+          if (this.doc?.getType() !== type) {
+            // TODO(24596): page reload after checking for pending ops?
+            throw new OError('ot type mismatch', {
+              got: type,
+              want: this.doc?.getType(),
+            })
+          }
+          if (type === 'sharejs-text-ot') {
+            this.decodeRanges(ranges)
+            this.catchUpRanges(ranges?.changes, ranges?.comments)
+          }
          callback?.()
        }
      )
@@ -464,8 +485,18 @@ export class DocumentContainer extends EventEmitter {
      this.socket.emit(
        'joinDoc',
        this.doc_id,
-        { encodeRanges: true },
-        (error, docLines, version, updates, ranges) => {
+        {
+          encodeRanges: true,
+          supportsHistoryV1OT: true,
+        },
+        (
+          error,
+          docLines,
+          version,
+          updates,
+          ranges,
+          type: OTType = 'sharejs-text-ot'
+        ) => {
          if (error) {
            callback?.(error)
            return
@@ -477,9 +508,12 @@ export class DocumentContainer extends EventEmitter {
            version,
            this.socket,
            this.globalEditorWatchdogManager,
-            this.ideEventEmitter
+            this.ideEventEmitter,
+            type
          )
-          this.decodeRanges(ranges)
+          if (type === 'sharejs-text-ot') {
+            this.decodeRanges(ranges)
+          }
          this.ranges = new RangesTracker(ranges?.changes, ranges?.comments)
          this.bindToShareJsDocEvents()
          callback?.()
@@ -580,7 +614,9 @@ export class DocumentContainer extends EventEmitter {
    this.doc.on(
      'change',
      (ops: AnyOperation[], oldSnapshot: any, msg: Message) => {
-        this.applyOpsToRanges(ops, msg)
+        if (this.getType() === 'sharejs-text-ot') {
+          this.applyOpsToRanges(ops, msg)
+        }
        if (docChangedTimeout) {
          window.clearTimeout(docChangedTimeout)
        }
@@ -2,7 +2,7 @@
 // Migrated from services/web/frontend/js/ide/editor/ShareJsDoc.js

 import EventEmitter from '../../../utils/EventEmitter'
-import { Doc } from '@/vendor/libs/sharejs'
+import sharejs, { Doc } from '@/vendor/libs/sharejs'
 import { Socket } from '@/features/ide-react/connection/types/socket'
 import { debugConsole } from '@/utils/debugging'
 import { decodeUtf8 } from '@/utils/decode-utf8'
@@ -12,11 +12,18 @@ import {
  Message,
  ShareJsConnectionState,
  ShareJsOperation,
+  ShareJsTextType,
  TrackChangesIdSeeds,
 } from '@/features/ide-react/editor/types/document'
 import { EditorFacade } from '@/features/source-editor/extensions/realtime'
 import { recordDocumentFirstChangeEvent } from '@/features/event-tracking/document-first-change-event'
 import getMeta from '@/utils/meta'
+import { HistoryOTType } from './share-js-history-ot-type'
+import { StringFileData } from 'overleaf-editor-core/index'
+import {
+  RawEditOperation,
+  StringFileRawData,
+} from 'overleaf-editor-core/lib/types'

 // All times below are in milliseconds
 const SINGLE_USER_FLUSH_DELAY = 2000
@@ -27,6 +34,7 @@ const FATAL_OP_TIMEOUT = 45000
 const RECENT_ACK_LIMIT = 2 * SINGLE_USER_FLUSH_DELAY

 type Update = Record<string, any>
+export type OTType = 'sharejs-text-ot' | 'history-ot'

 type Connection = {
  send: (update: Update) => void
@@ -35,7 +43,6 @@ type Connection = {
 }

 export class ShareJsDoc extends EventEmitter {
-  type: string
  track_changes = false
  track_changes_id_seeds: TrackChangesIdSeeds | null = null
  connection: Connection
@@ -57,12 +64,24 @@ export class ShareJsDoc extends EventEmitter {
    version: number,
    readonly socket: Socket,
    private readonly globalEditorWatchdogManager: EditorWatchdogManager,
-    private readonly eventEmitter: IdeEventEmitter
+    private readonly eventEmitter: IdeEventEmitter,
+    readonly type: OTType = 'sharejs-text-ot'
  ) {
    super()
-    this.type = 'text'
+    let sharejsType: ShareJsTextType = sharejs.types.text
    // Decode any binary bits of data
-    const snapshot = docLines.map(line => decodeUtf8(line)).join('\n')
+    let snapshot: string | StringFileData
+    if (this.type === 'history-ot') {
+      snapshot = StringFileData.fromRaw(
+        docLines as unknown as StringFileRawData
+      )
+      sharejsType = new HistoryOTType(snapshot) as ShareJsTextType<
+        StringFileData,
+        RawEditOperation[]
+      >
+    } else {
+      snapshot = docLines.map(line => decodeUtf8(line)).join('\n')
+    }

    this.connection = {
      send: (update: Update) => {
@@ -89,7 +108,7 @@ export class ShareJsDoc extends EventEmitter {
    }

    this._doc = new Doc(this.connection, this.doc_id, {
-      type: this.type,
+      type: sharejsType,
    })
    this._doc.setFlushDelay(SINGLE_USER_FLUSH_DELAY)
    this._doc.on('change', (...args: any[]) => {
@@ -0,0 +1,131 @@
+import EventEmitter from '@/utils/EventEmitter'
+import {
+  EditOperationBuilder,
+  InsertOp,
+  RemoveOp,
+  RetainOp,
+  StringFileData,
+  TextOperation,
+} from 'overleaf-editor-core'
+import { RawEditOperation } from 'overleaf-editor-core/lib/types'
+
+function loadTextOperation(raw: RawEditOperation): TextOperation {
+  const operation = EditOperationBuilder.fromJSON(raw)
+  if (!(operation instanceof TextOperation)) {
+    throw new Error(`operation not supported: ${operation.constructor.name}`)
+  }
+  return operation
+}
+
+export class HistoryOTType extends EventEmitter {
+  // stub interface, these are actually on the Doc
+  api: HistoryOTType
+  snapshot: StringFileData
+
+  constructor(snapshot: StringFileData) {
+    super()
+    this.api = this
+    this.snapshot = snapshot
+  }
+
+  transformX(raw1: RawEditOperation[], raw2: RawEditOperation[]) {
+    const [a, b] = TextOperation.transform(
+      loadTextOperation(raw1[0]),
+      loadTextOperation(raw2[0])
+    )
+    return [[a.toJSON()], [b.toJSON()]]
+  }
+
+  apply(snapshot: StringFileData, rawEditOperation: RawEditOperation[]) {
+    const operation = loadTextOperation(rawEditOperation[0])
+    const afterFile = StringFileData.fromRaw(snapshot.toRaw())
+    afterFile.edit(operation)
+    this.snapshot = afterFile
+    return afterFile
+  }
+
+  compose(op1: RawEditOperation[], op2: RawEditOperation[]) {
+    return [
+      loadTextOperation(op1[0]).compose(loadTextOperation(op2[0])).toJSON(),
+    ]
+  }
+
+  // Do not provide normalize, used by submitOp to fixup bad input.
+  // normalize(op: TextOperation) {}
+
+  // Do not provide invert, only needed for reverting a rejected update.
+  // We are displaying an out-of-sync modal when an op is rejected.
+  // invert(op: TextOperation) {}
+
+  // API
+  insert(pos: number, text: string, fromUndo: boolean) {
+    const old = this.getText()
+    const op = new TextOperation()
+    op.retain(pos)
+    op.insert(text)
+    op.retain(old.length - pos)
+    this.submitOp([op.toJSON()])
+  }
+
+  del(pos: number, length: number, fromUndo: boolean) {
+    const old = this.getText()
+    const op = new TextOperation()
+    op.retain(pos)
+    op.remove(length)
+    op.retain(old.length - pos - length)
+    this.submitOp([op.toJSON()])
+  }
+
+  getText() {
+    return this.snapshot.getContent({ filterTrackedDeletes: true })
+  }
+
+  getLength() {
+    return this.getText().length
+  }
+
+  _register() {
+    this.on(
+      'remoteop',
+      (rawEditOperation: RawEditOperation[], oldSnapshot: StringFileData) => {
+        const operation = loadTextOperation(rawEditOperation[0])
+        const str = oldSnapshot.getContent()
+        if (str.length !== operation.baseLength)
+          throw new TextOperation.ApplyError(
+            "The operation's base length must be equal to the string's length.",
+            operation,
+            str
+          )
+
+        let outputCursor = 0
+        let inputCursor = 0
+        for (const op of operation.ops) {
+          if (op instanceof RetainOp) {
+            inputCursor += op.length
+            outputCursor += op.length
+          } else if (op instanceof InsertOp) {
+            this.emit('insert', outputCursor, op.insertion, op.insertion.length)
+            outputCursor += op.insertion.length
+          } else if (op instanceof RemoveOp) {
+            this.emit(
+              'delete',
+              outputCursor,
+              str.slice(inputCursor, inputCursor + op.length)
+            )
+            inputCursor += op.length
+          }
+        }
+
+        if (inputCursor !== str.length)
+          throw new TextOperation.ApplyError(
+            "The operation didn't operate on the whole string.",
+            operation,
+            str
+          )
+      }
+    )
+  }
+
+  // stub-interface, provided by sharejs.Doc
+  submitOp(op: RawEditOperation[]) {}
+}
@@ -1,3 +1,4 @@
+import { StringFileData } from 'overleaf-editor-core'
 import { AnyOperation } from '../../../../../../types/change'

 export type Version = number
@@ -8,6 +9,23 @@ export type ShareJsOperation = AnyOperation[]

 export type TrackChangesIdSeeds = { inflight: string; pending: string }

+export interface ShareJsTextType<Snapshot = any, Operation = any> {
+  transformX(op1: Operation, op2: Operation): Operation[]
+  apply(snapshot: Snapshot, op: Operation): Snapshot
+  compose(op1: Operation, op2: Operation): Operation
+
+  api: {
+    insert(pos: number, text: string, fromUndo: boolean): void
+    del(pos: number, length: number, fromUndo: boolean): void
+    getText(): string
+    getLength(): number
+    _register(): void
+  }
+
+  // stub-interface, provided by sharejs.Doc
+  submitOp(op: Operation): void
+}
+
 // TODO: check the properties of this type
 export type Message = {
  v: Version
@@ -16,5 +34,6 @@ export type Message = {
    type?: string
  }
  doc?: string
-  snapshot?: string
+  snapshot?: string | StringFileData
+  type?: ShareJsTextType
 }