Merge pull request #2106 from TriliumNext/fix/llm-becca-sync

fix(llm): Fix Note Embeddings not being synced correctly and causing sync loops

apps/client/src/services/froca_updater.ts

@@ -35,8 +35,8 @@ async function processEntityChanges(entityChanges: EntityChange[]) {
                 loadResults.addOption(attributeEntity.name);
             } else if (ec.entityName === "attachments") {
                 processAttachment(loadResults, ec);
-            } else if (ec.entityName === "blobs" || ec.entityName === "etapi_tokens") {
-                // NOOP
+            } else if (ec.entityName === "blobs" || ec.entityName === "etapi_tokens" || ec.entityName === "note_embeddings") {
+                // NOOP - these entities are handled at the backend level and don't require frontend processing
             } else {
                 throw new Error(`Unknown entityName '${ec.entityName}'`);
             }

apps/client/src/services/load_results.ts

@@ -44,9 +44,17 @@ interface OptionRow {}
 
 interface NoteReorderingRow {}
 
-interface ContentNoteIdToComponentIdRow {
+interface NoteEmbeddingRow {
+    embedId: string;
     noteId: string;
-    componentId: string;
+    providerId: string;
+    modelId: string;
+    dimension: number;
+    version: number;
+    dateCreated: string;
+    utcDateCreated: string;
+    dateModified: string;
+    utcDateModified: string;
 }
 
 type EntityRowMappings = {
@@ -56,6 +64,7 @@ type EntityRowMappings = {
     options: OptionRow;
     revisions: RevisionRow;
     note_reordering: NoteReorderingRow;
+    note_embeddings: NoteEmbeddingRow;
 };
 
 export type EntityRowNames = keyof EntityRowMappings;

apps/server/src/becca/becca-interface.ts

@@ -12,6 +12,7 @@ import type { AttachmentRow, BlobRow, RevisionRow } from "@triliumnext/commons";
 import BBlob from "./entities/bblob.js";
 import BRecentNote from "./entities/brecent_note.js";
 import type AbstractBeccaEntity from "./entities/abstract_becca_entity.js";
+import type BNoteEmbedding from "./entities/bnote_embedding.js";
 
 interface AttachmentOpts {
     includeContentLength?: boolean;
@@ -32,6 +33,7 @@ export default class Becca {
     attributeIndex!: Record<string, BAttribute[]>;
     options!: Record<string, BOption>;
     etapiTokens!: Record<string, BEtapiToken>;
+    noteEmbeddings!: Record<string, BNoteEmbedding>;
 
     allNoteSetCache: NoteSet | null;
 
@@ -48,6 +50,7 @@ export default class Becca {
         this.attributeIndex = {};
         this.options = {};
         this.etapiTokens = {};
+        this.noteEmbeddings = {};
 
         this.dirtyNoteSetCache();
 

apps/server/src/becca/becca_loader.ts

@@ -9,9 +9,10 @@ import BBranch from "./entities/bbranch.js";
 import BAttribute from "./entities/battribute.js";
 import BOption from "./entities/boption.js";
 import BEtapiToken from "./entities/betapi_token.js";
+import BNoteEmbedding from "./entities/bnote_embedding.js";
 import cls from "../services/cls.js";
 import entityConstructor from "../becca/entity_constructor.js";
-import type { AttributeRow, BranchRow, EtapiTokenRow, NoteRow, OptionRow } from "@triliumnext/commons";
+import type { AttributeRow, BranchRow, EtapiTokenRow, NoteRow, OptionRow, NoteEmbeddingRow } from "@triliumnext/commons";
 import type AbstractBeccaEntity from "./entities/abstract_becca_entity.js";
 import ws from "../services/ws.js";
 
@@ -63,6 +64,10 @@ function load() {
         for (const row of sql.getRows<EtapiTokenRow>(/*sql*/`SELECT etapiTokenId, name, tokenHash, utcDateCreated, utcDateModified FROM etapi_tokens WHERE isDeleted = 0`)) {
             new BEtapiToken(row);
         }
+
+        for (const row of sql.getRows<NoteEmbeddingRow>(/*sql*/`SELECT embedId, noteId, providerId, modelId, dimension, embedding, version, dateCreated, dateModified, utcDateCreated, utcDateModified FROM note_embeddings`)) {
+            new BNoteEmbedding(row).init();
+        }
     });
 
     for (const noteId in becca.notes) {
@@ -85,7 +90,7 @@ eventService.subscribeBeccaLoader([eventService.ENTITY_CHANGE_SYNCED], ({ entity
         return;
     }
 
-    if (["notes", "branches", "attributes", "etapi_tokens", "options"].includes(entityName)) {
+    if (["notes", "branches", "attributes", "etapi_tokens", "options", "note_embeddings"].includes(entityName)) {
         const EntityClass = entityConstructor.getEntityFromEntityName(entityName);
         const primaryKeyName = EntityClass.primaryKeyName;
 
@@ -143,6 +148,8 @@ eventService.subscribeBeccaLoader([eventService.ENTITY_DELETED, eventService.ENT
         attributeDeleted(entityId);
     } else if (entityName === "etapi_tokens") {
         etapiTokenDeleted(entityId);
+    } else if (entityName === "note_embeddings") {
+        noteEmbeddingDeleted(entityId);
     }
 });
 
@@ -278,6 +285,10 @@ function etapiTokenDeleted(etapiTokenId: string) {
     delete becca.etapiTokens[etapiTokenId];
 }
 
+function noteEmbeddingDeleted(embedId: string) {
+    delete becca.noteEmbeddings[embedId];
+}
+
 eventService.subscribeBeccaLoader(eventService.ENTER_PROTECTED_SESSION, () => {
     try {
         becca.decryptProtectedNotes();

apps/server/src/becca/entities/bnote_embedding.ts

@@ -32,6 +32,12 @@ class BNoteEmbedding extends AbstractBeccaEntity<BNoteEmbedding> {
         }
     }
 
+    init() {
+        if (this.embedId) {
+            this.becca.noteEmbeddings[this.embedId] = this;
+        }
+    }
+
     updateFromRow(row: NoteEmbeddingRow): void {
         this.embedId = row.embedId;
         this.noteId = row.noteId;
@@ -44,6 +50,10 @@ class BNoteEmbedding extends AbstractBeccaEntity<BNoteEmbedding> {
         this.dateModified = row.dateModified;
         this.utcDateCreated = row.utcDateCreated;
         this.utcDateModified = row.utcDateModified;
+
+        if (this.embedId) {
+            this.becca.noteEmbeddings[this.embedId] = this;
+        }
     }
 
     override beforeSaving() {

apps/server/src/services/consistency_checks.ts

@@ -799,6 +799,7 @@ class ConsistencyChecks {
         this.runEntityChangeChecks("attributes", "attributeId");
         this.runEntityChangeChecks("etapi_tokens", "etapiTokenId");
         this.runEntityChangeChecks("options", "name");
+        this.runEntityChangeChecks("note_embeddings", "embedId");
     }
 
     findWronglyNamedAttributes() {

apps/server/src/services/ws.ts

@@ -203,6 +203,13 @@ function fillInAdditionalProperties(entityChange: EntityChange) {
                                                 WHERE attachmentId = ?`,
             [entityChange.entityId]
         );
+    } else if (entityChange.entityName === "note_embeddings") {
+        // Note embeddings are backend-only entities for AI/vector search
+        // Frontend doesn't need the full embedding data (which is large binary data)
+        // Just ensure entity is marked as handled - actual sync happens at database level
+        if (!entityChange.isErased) {
+            entityChange.entity = { embedId: entityChange.entityId };
+        }
     }
 
     if (entityChange.entity instanceof AbstractBeccaEntity) {