Publish camino-datatypes from ce2c8b6ed1926da7b3b3fe4aa3f1bbd8f327470a

.quixos-subtree-source.json

@@ -1,7 +1,7 @@
 {
   "version": 1,
   "sourceRepo": "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos.git",
-  "sourceCommit": "5ae087c6c8f9c945d116858341ec882ab4c72636",
+  "sourceCommit": "ce2c8b6ed1926da7b3b3fe4aa3f1bbd8f327470a",
   "sourcePath": "quixos-instance/packages/camino-datatypes",
   "exportName": "camino-datatypes",
   "mirrorRemote": "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/camino-datatypes.git"

dist/src/crdt-automerge.d.ts

@@ -1,5 +1,7 @@
 import * as Automerge from "@automerge/automerge";
-export type CaminoCrdtEncoding = "base64";
+export declare const AUTOMERGE_SNAPSHOT_ENCODING = "automerge-snapshot-v1";
+export declare const AUTOMERGE_CHANGES_ENCODING = "automerge-changes-v1";
+export type CaminoCrdtEncoding = "base64" | typeof AUTOMERGE_SNAPSHOT_ENCODING | typeof AUTOMERGE_CHANGES_ENCODING;
 export type CaminoCrdtEnvelope = {
     $caminoCrdtType: string;
     $caminoCrdtEncoding: CaminoCrdtEncoding;
@@ -16,6 +18,8 @@ export declare const loadAutomergeFieldDoc: (value: CaminoCrdtEnvelope, type: st
 export declare const saveAutomergeFieldDoc: (doc: Automerge.Doc<AutomergeFieldDoc>, type: string) => CaminoCrdtEnvelope;
 export declare const createAutomergeFieldValue: (value: unknown, type: string) => CaminoCrdtEnvelope;
 export declare const isAutomergeFieldDoc: (value: unknown) => value is Automerge.Doc<AutomergeFieldDoc>;
+export declare const reconcileAutomergeFieldDocs: <T>(base: CaminoAutomergeDoc<T>, desired: CaminoAutomergeDoc<T>) => CaminoAutomergeDoc<T>;
+export declare const createAutomergeFieldChanges: (base: Automerge.Doc<AutomergeFieldDoc>, desired: Automerge.Doc<AutomergeFieldDoc>, type: string) => CaminoCrdtEnvelope;
 export declare const encodeAutomergeFieldWrite: (value: unknown, type: string) => CaminoCrdtEnvelope;
 export declare const normalizeAutomergeFieldValue: (value: unknown, type: string) => CaminoCrdtEnvelope;
 export declare const materializeAutomergeFieldValue: (value: CaminoCrdtEnvelope, type: string) => unknown;

dist/src/crdt-automerge.js

@@ -1,4 +1,6 @@
 import * as Automerge from "@automerge/automerge";
+export const AUTOMERGE_SNAPSHOT_ENCODING = "automerge-snapshot-v1";
+export const AUTOMERGE_CHANGES_ENCODING = "automerge-changes-v1";
 const bytesToBase64 = (value) => {
     if (typeof Buffer !== "undefined") {
         return Buffer.from(value).toString("base64");
@@ -11,7 +13,7 @@ const bytesToBase64 = (value) => {
 };
 const base64ToBytes = (value) => {
     if (typeof Buffer !== "undefined") {
-        return Buffer.from(value, "base64");
+        return new Uint8Array(Buffer.from(value, "base64"));
     }
     const binary = atob(value);
     const bytes = new Uint8Array(binary.length);
@@ -20,6 +22,9 @@ const base64ToBytes = (value) => {
     }
     return bytes;
 };
+const isCrdtEncoding = (value) => value === "base64" ||
+    value === AUTOMERGE_SNAPSHOT_ENCODING ||
+    value === AUTOMERGE_CHANGES_ENCODING;
 export const isCaminoCrdtEnvelope = (value) => Boolean(value) &&
     typeof value === "object" &&
     !Array.isArray(value) &&
@@ -27,22 +32,27 @@ export const isCaminoCrdtEnvelope = (value) => Boolean(value) &&
         "string" &&
     typeof value.$caminoCrdtPayload ===
         "string" &&
-    (value.$caminoCrdtEncoding ===
-        undefined ||
-        value.$caminoCrdtEncoding ===
-            "base64");
+    isCrdtEncoding(value.$caminoCrdtEncoding ??
+        "base64");
 const assertEnvelopeType = (value, type) => {
     if (value.$caminoCrdtType !== type) {
         throw new Error(`CRDT type mismatch: expected ${type}, got ${value.$caminoCrdtType}`);
     }
 };
-export const loadAutomergeFieldDoc = (value, type) => {
+const requireSnapshotEnvelope = (value, type) => {
     assertEnvelopeType(value, type);
-    return Automerge.load(base64ToBytes(value.$caminoCrdtPayload));
+    if (value.$caminoCrdtEncoding === AUTOMERGE_CHANGES_ENCODING) {
+        throw new Error("An Automerge change batch cannot be loaded without a base document");
+    }
+    return value;
+};
+export const loadAutomergeFieldDoc = (value, type) => {
+    const snapshot = requireSnapshotEnvelope(value, type);
+    return Automerge.load(base64ToBytes(snapshot.$caminoCrdtPayload));
 };
 export const saveAutomergeFieldDoc = (doc, type) => ({
     $caminoCrdtType: type,
-    $caminoCrdtEncoding: "base64",
+    $caminoCrdtEncoding: AUTOMERGE_SNAPSHOT_ENCODING,
     $caminoCrdtPayload: bytesToBase64(Automerge.save(doc)),
 });
 export const createAutomergeFieldValue = (value, type) => saveAutomergeFieldDoc(Automerge.from({ value }), type);
@@ -58,6 +68,56 @@ export const isAutomergeFieldDoc = (value) => {
         return false;
     }
 };
+const encodeChanges = (changes) => {
+    const totalLength = changes.reduce((total, change) => total + 4 + change.length, 4);
+    const payload = new Uint8Array(totalLength);
+    const view = new DataView(payload.buffer);
+    view.setUint32(0, changes.length);
+    let offset = 4;
+    for (const change of changes) {
+        view.setUint32(offset, change.length);
+        offset += 4;
+        payload.set(change, offset);
+        offset += change.length;
+    }
+    return payload;
+};
+const decodeChanges = (payload) => {
+    if (payload.length < 4) {
+        throw new Error("Invalid Automerge change batch: missing change count");
+    }
+    const view = new DataView(payload.buffer, payload.byteOffset, payload.byteLength);
+    const count = view.getUint32(0);
+    const changes = [];
+    let offset = 4;
+    for (let index = 0; index < count; index += 1) {
+        if (offset + 4 > payload.length) {
+            throw new Error("Invalid Automerge change batch: missing change length");
+        }
+        const length = view.getUint32(offset);
+        offset += 4;
+        if (offset + length > payload.length) {
+            throw new Error("Invalid Automerge change batch: truncated change");
+        }
+        changes.push(payload.slice(offset, offset + length));
+        offset += length;
+    }
+    if (offset !== payload.length) {
+        throw new Error("Invalid Automerge change batch: trailing bytes");
+    }
+    return changes;
+};
+export const reconcileAutomergeFieldDocs = (base, desired) => Automerge.merge(Automerge.clone(base), Automerge.clone(desired));
+export const createAutomergeFieldChanges = (base, desired, type) => {
+    const next = reconcileAutomergeFieldDocs(base, desired);
+    return {
+        $caminoCrdtType: type,
+        $caminoCrdtEncoding: AUTOMERGE_CHANGES_ENCODING,
+        $caminoCrdtPayload: bytesToBase64(encodeChanges(Automerge.getChanges(base, next))),
+    };
+};
+// Full snapshots remain an explicit escape hatch for initialization, import,
+// and recovery when a client cannot establish a shared Automerge history.
 export const encodeAutomergeFieldWrite = (value, type) => {
     if (isCaminoCrdtEnvelope(value)) {
         return saveAutomergeFieldDoc(loadAutomergeFieldDoc(value, type), type);
@@ -70,10 +130,25 @@ export const encodeAutomergeFieldWrite = (value, type) => {
 export const normalizeAutomergeFieldValue = (value, type) => encodeAutomergeFieldWrite(value, type);
 export const materializeAutomergeFieldValue = (value, type) => loadAutomergeFieldDoc(value, type).value;
 export const mergeAutomergeFieldValues = ({ existing, incoming, type, }) => {
-    const incomingDoc = loadAutomergeFieldDoc(normalizeAutomergeFieldValue(incoming, type), type);
-    const mergedDoc = existing === undefined || existing === null
-        ? incomingDoc
-        : Automerge.merge(loadAutomergeFieldDoc(normalizeAutomergeFieldValue(existing, type), type), incomingDoc);
+    const incomingEnvelope = isCaminoCrdtEnvelope(incoming)
+        ? incoming
+        : encodeAutomergeFieldWrite(incoming, type);
+    assertEnvelopeType(incomingEnvelope, type);
+    let mergedDoc;
+    if (incomingEnvelope.$caminoCrdtEncoding === AUTOMERGE_CHANGES_ENCODING) {
+        if (existing === undefined || existing === null) {
+            throw new Error("Cannot apply an Automerge change batch to a missing field");
+        }
+        const existingDoc = loadAutomergeFieldDoc(encodeAutomergeFieldWrite(existing, type), type);
+        [mergedDoc] = Automerge.applyChanges(existingDoc, decodeChanges(base64ToBytes(incomingEnvelope.$caminoCrdtPayload)));
+    }
+    else {
+        const incomingDoc = loadAutomergeFieldDoc(incomingEnvelope, type);
+        mergedDoc =
+            existing === undefined || existing === null
+                ? incomingDoc
+                : Automerge.merge(loadAutomergeFieldDoc(encodeAutomergeFieldWrite(existing, type), type), incomingDoc);
+    }
     const stored = saveAutomergeFieldDoc(mergedDoc, type);
     return {
         stored,

src/crdt-automerge.ts

@@ -1,6 +1,13 @@
 import * as Automerge from "@automerge/automerge";
 
-export type CaminoCrdtEncoding = "base64";
+export const AUTOMERGE_SNAPSHOT_ENCODING = "automerge-snapshot-v1";
+export const AUTOMERGE_CHANGES_ENCODING = "automerge-changes-v1";
+
+// `base64` was the original encoding label. Its payload was an Automerge save.
+export type CaminoCrdtEncoding =
+  | "base64"
+  | typeof AUTOMERGE_SNAPSHOT_ENCODING
+  | typeof AUTOMERGE_CHANGES_ENCODING;
 
 export type CaminoCrdtEnvelope = {
   $caminoCrdtType: string;
@@ -29,7 +36,7 @@ const bytesToBase64 = (value: Uint8Array) => {
 
 const base64ToBytes = (value: string) => {
   if (typeof Buffer !== "undefined") {
-    return Buffer.from(value, "base64");
+    return new Uint8Array(Buffer.from(value, "base64"));
   }
   const binary = atob(value);
   const bytes = new Uint8Array(binary.length);
@@ -39,6 +46,11 @@ const base64ToBytes = (value: string) => {
   return bytes;
 };
 
+const isCrdtEncoding = (value: unknown): value is CaminoCrdtEncoding =>
+  value === "base64" ||
+  value === AUTOMERGE_SNAPSHOT_ENCODING ||
+  value === AUTOMERGE_CHANGES_ENCODING;
+
 export const isCaminoCrdtEnvelope = (
   value: unknown,
 ): value is CaminoCrdtEnvelope =>
@@ -49,10 +61,10 @@ export const isCaminoCrdtEnvelope = (
     "string" &&
   typeof (value as { $caminoCrdtPayload?: unknown }).$caminoCrdtPayload ===
     "string" &&
-  ((value as { $caminoCrdtEncoding?: unknown }).$caminoCrdtEncoding ===
-    undefined ||
-    (value as { $caminoCrdtEncoding?: unknown }).$caminoCrdtEncoding ===
-      "base64");
+  isCrdtEncoding(
+    (value as { $caminoCrdtEncoding?: unknown }).$caminoCrdtEncoding ??
+      "base64",
+  );
 
 const assertEnvelopeType = (value: CaminoCrdtEnvelope, type: string) => {
   if (value.$caminoCrdtType !== type) {
@@ -62,13 +74,21 @@ const assertEnvelopeType = (value: CaminoCrdtEnvelope, type: string) => {
   }
 };
 
+const requireSnapshotEnvelope = (value: CaminoCrdtEnvelope, type: string) => {
+  assertEnvelopeType(value, type);
+  if (value.$caminoCrdtEncoding === AUTOMERGE_CHANGES_ENCODING) {
+    throw new Error("An Automerge change batch cannot be loaded without a base document");
+  }
+  return value;
+};
+
 export const loadAutomergeFieldDoc = (
   value: CaminoCrdtEnvelope,
   type: string,
 ): Automerge.Doc<AutomergeFieldDoc> => {
-  assertEnvelopeType(value, type);
+  const snapshot = requireSnapshotEnvelope(value, type);
   return Automerge.load<AutomergeFieldDoc>(
-    base64ToBytes(value.$caminoCrdtPayload),
+    base64ToBytes(snapshot.$caminoCrdtPayload),
   );
 };
 
@@ -77,7 +97,7 @@ export const saveAutomergeFieldDoc = (
   type: string,
 ): CaminoCrdtEnvelope => ({
   $caminoCrdtType: type,
-  $caminoCrdtEncoding: "base64",
+  $caminoCrdtEncoding: AUTOMERGE_SNAPSHOT_ENCODING,
   $caminoCrdtPayload: bytesToBase64(Automerge.save(doc)),
 });
 
@@ -101,6 +121,70 @@ export const isAutomergeFieldDoc = (
   }
 };
 
+const encodeChanges = (changes: Automerge.Change[]): Uint8Array => {
+  const totalLength = changes.reduce((total, change) => total + 4 + change.length, 4);
+  const payload = new Uint8Array(totalLength);
+  const view = new DataView(payload.buffer);
+  view.setUint32(0, changes.length);
+  let offset = 4;
+  for (const change of changes) {
+    view.setUint32(offset, change.length);
+    offset += 4;
+    payload.set(change, offset);
+    offset += change.length;
+  }
+  return payload;
+};
+
+const decodeChanges = (payload: Uint8Array): Automerge.Change[] => {
+  if (payload.length < 4) {
+    throw new Error("Invalid Automerge change batch: missing change count");
+  }
+  const view = new DataView(payload.buffer, payload.byteOffset, payload.byteLength);
+  const count = view.getUint32(0);
+  const changes: Automerge.Change[] = [];
+  let offset = 4;
+  for (let index = 0; index < count; index += 1) {
+    if (offset + 4 > payload.length) {
+      throw new Error("Invalid Automerge change batch: missing change length");
+    }
+    const length = view.getUint32(offset);
+    offset += 4;
+    if (offset + length > payload.length) {
+      throw new Error("Invalid Automerge change batch: truncated change");
+    }
+    changes.push(payload.slice(offset, offset + length));
+    offset += length;
+  }
+  if (offset !== payload.length) {
+    throw new Error("Invalid Automerge change batch: trailing bytes");
+  }
+  return changes;
+};
+
+export const reconcileAutomergeFieldDocs = <T>(
+  base: CaminoAutomergeDoc<T>,
+  desired: CaminoAutomergeDoc<T>,
+): CaminoAutomergeDoc<T> =>
+  Automerge.merge(Automerge.clone(base), Automerge.clone(desired));
+
+export const createAutomergeFieldChanges = (
+  base: Automerge.Doc<AutomergeFieldDoc>,
+  desired: Automerge.Doc<AutomergeFieldDoc>,
+  type: string,
+): CaminoCrdtEnvelope => {
+  const next = reconcileAutomergeFieldDocs(base, desired);
+  return {
+    $caminoCrdtType: type,
+    $caminoCrdtEncoding: AUTOMERGE_CHANGES_ENCODING,
+    $caminoCrdtPayload: bytesToBase64(
+      encodeChanges(Automerge.getChanges(base, next)),
+    ),
+  };
+};
+
+// Full snapshots remain an explicit escape hatch for initialization, import,
+// and recovery when a client cannot establish a shared Automerge history.
 export const encodeAutomergeFieldWrite = (
   value: unknown,
   type: string,
@@ -142,17 +226,35 @@ export const mergeAutomergeFieldValues = ({
   incoming,
   type,
 }: MergeAutomergeFieldValueParams): MergeAutomergeFieldValueResult => {
-  const incomingDoc = loadAutomergeFieldDoc(
-    normalizeAutomergeFieldValue(incoming, type),
-    type,
-  );
-  const mergedDoc =
-    existing === undefined || existing === null
-      ? incomingDoc
-      : Automerge.merge(
-          loadAutomergeFieldDoc(normalizeAutomergeFieldValue(existing, type), type),
-          incomingDoc,
-        );
+  const incomingEnvelope = isCaminoCrdtEnvelope(incoming)
+    ? incoming
+    : encodeAutomergeFieldWrite(incoming, type);
+  assertEnvelopeType(incomingEnvelope, type);
+
+  let mergedDoc: Automerge.Doc<AutomergeFieldDoc>;
+  if (incomingEnvelope.$caminoCrdtEncoding === AUTOMERGE_CHANGES_ENCODING) {
+    if (existing === undefined || existing === null) {
+      throw new Error("Cannot apply an Automerge change batch to a missing field");
+    }
+    const existingDoc = loadAutomergeFieldDoc(
+      encodeAutomergeFieldWrite(existing, type),
+      type,
+    );
+    [mergedDoc] = Automerge.applyChanges(
+      existingDoc,
+      decodeChanges(base64ToBytes(incomingEnvelope.$caminoCrdtPayload)),
+    );
+  } else {
+    const incomingDoc = loadAutomergeFieldDoc(incomingEnvelope, type);
+    mergedDoc =
+      existing === undefined || existing === null
+        ? incomingDoc
+        : Automerge.merge(
+            loadAutomergeFieldDoc(encodeAutomergeFieldWrite(existing, type), type),
+            incomingDoc,
+          );
+  }
+
   const stored = saveAutomergeFieldDoc(mergedDoc, type);
   return {
     stored,

test/crdt-automerge.test.ts

@@ -2,12 +2,16 @@ import assert from "node:assert/strict";
 import test from "node:test";
 import * as Automerge from "@automerge/automerge";
 import {
+  AUTOMERGE_CHANGES_ENCODING,
+  AUTOMERGE_SNAPSHOT_ENCODING,
   createAutomergeFieldValue,
+  createAutomergeFieldChanges,
   encodeAutomergeFieldWrite,
   isAutomergeFieldDoc,
   loadAutomergeFieldDoc,
   materializeAutomergeFieldValue,
   mergeAutomergeFieldValues,
+  reconcileAutomergeFieldDocs,
   saveAutomergeFieldDoc,
 } from "../src/crdt-automerge.js";
 
@@ -17,7 +21,7 @@ test("Automerge CRDT fields explicitly create stored envelopes", () => {
   const stored = createAutomergeFieldValue({ title: "draft" }, type);
 
   assert.equal(stored.$caminoCrdtType, type);
-  assert.equal(stored.$caminoCrdtEncoding, "base64");
+  assert.equal(stored.$caminoCrdtEncoding, AUTOMERGE_SNAPSHOT_ENCODING);
   assert.deepEqual(materializeAutomergeFieldValue(stored, type), {
     title: "draft",
   });
@@ -72,3 +76,75 @@ test("Automerge CRDT fields merge divergent saved documents", () => {
     right: true,
   });
 });
+
+test("Automerge CRDT fields encode and apply incremental changes", () => {
+  const stored = createAutomergeFieldValue({ title: "draft" }, type);
+  const base = loadAutomergeFieldDoc(stored, type);
+  const changed = Automerge.change(base, (doc) => {
+    (doc.value as { title: string }).title = "edited";
+  });
+  const update = createAutomergeFieldChanges(base, changed, type);
+
+  assert.equal(update.$caminoCrdtEncoding, AUTOMERGE_CHANGES_ENCODING);
+  assert.ok(update.$caminoCrdtPayload.length < saveAutomergeFieldDoc(changed, type).$caminoCrdtPayload.length);
+
+  const result = mergeAutomergeFieldValues({ existing: stored, incoming: update, type });
+  assert.deepEqual(result.materialized, { title: "edited" });
+  assert.equal(result.stored.$caminoCrdtEncoding, AUTOMERGE_SNAPSHOT_ENCODING);
+});
+
+test("Automerge change application is idempotent", () => {
+  const stored = createAutomergeFieldValue({ count: 0 }, type);
+  const base = loadAutomergeFieldDoc(stored, type);
+  const changed = Automerge.change(base, (doc) => {
+    (doc.value as { count: number }).count = 1;
+  });
+  const update = createAutomergeFieldChanges(base, changed, type);
+
+  const once = mergeAutomergeFieldValues({ existing: stored, incoming: update, type });
+  const twice = mergeAutomergeFieldValues({ existing: once.stored, incoming: update, type });
+  assert.deepEqual(twice.materialized, { count: 1 });
+});
+
+test("Automerge changes retain remote history while local writes are pending", () => {
+  const stored = createAutomergeFieldValue({ local: false, remote: false }, type);
+  const base = loadAutomergeFieldDoc(stored, type);
+  const local = Automerge.change(
+    Automerge.clone(base, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
+    (doc) => {
+      (doc.value as { local: boolean }).local = true;
+    },
+  );
+  const remote = Automerge.change(
+    Automerge.clone(base, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"),
+    (doc) => {
+      (doc.value as { remote: boolean }).remote = true;
+    },
+  );
+  const rebased = reconcileAutomergeFieldDocs(remote, local);
+  const update = createAutomergeFieldChanges(remote, rebased, type);
+  const result = mergeAutomergeFieldValues({
+    existing: saveAutomergeFieldDoc(remote, type),
+    incoming: update,
+    type,
+  });
+
+  assert.deepEqual(result.materialized, { local: true, remote: true });
+});
+
+test("Automerge change batches reject malformed framing", () => {
+  const stored = createAutomergeFieldValue({ count: 0 }, type);
+  assert.throws(
+    () =>
+      mergeAutomergeFieldValues({
+        existing: stored,
+        incoming: {
+          $caminoCrdtType: type,
+          $caminoCrdtEncoding: AUTOMERGE_CHANGES_ENCODING,
+          $caminoCrdtPayload: Buffer.from([0, 0, 0, 1]).toString("base64"),
+        },
+        type,
+      }),
+    /missing change length/,
+  );
+});