From f3a21cda98c12b2a22c7f6994009bcd1e9146a6f Mon Sep 17 00:00:00 2001 From: perf3ct Date: Mon, 2 Jun 2025 15:26:32 +0000 Subject: [PATCH 1/4] fix(llm): add the noteEmbedding object to all things becca to make it happy --- apps/server/src/becca/becca-interface.ts | 3 +++ apps/server/src/becca/becca_loader.ts | 15 +++++++++++++-- apps/server/src/becca/entities/bnote_embedding.ts | 10 ++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/apps/server/src/becca/becca-interface.ts b/apps/server/src/becca/becca-interface.ts index 005a5cc52..4301b2b5e 100644 --- a/apps/server/src/becca/becca-interface.ts +++ b/apps/server/src/becca/becca-interface.ts @@ -12,6 +12,7 @@ import type { AttachmentRow, BlobRow, RevisionRow } from "@triliumnext/commons"; import BBlob from "./entities/bblob.js"; import BRecentNote from "./entities/brecent_note.js"; import type AbstractBeccaEntity from "./entities/abstract_becca_entity.js"; +import type BNoteEmbedding from "./entities/bnote_embedding.js"; interface AttachmentOpts { includeContentLength?: boolean; @@ -32,6 +33,7 @@ export default class Becca { attributeIndex!: Record; options!: Record; etapiTokens!: Record; + noteEmbeddings!: Record; allNoteSetCache: NoteSet | null; @@ -48,6 +50,7 @@ export default class Becca { this.attributeIndex = {}; this.options = {}; this.etapiTokens = {}; + this.noteEmbeddings = {}; this.dirtyNoteSetCache(); diff --git a/apps/server/src/becca/becca_loader.ts b/apps/server/src/becca/becca_loader.ts index 4506c912a..44e3a9ce2 100644 --- a/apps/server/src/becca/becca_loader.ts +++ b/apps/server/src/becca/becca_loader.ts @@ -9,9 +9,10 @@ import BBranch from "./entities/bbranch.js"; import BAttribute from "./entities/battribute.js"; import BOption from "./entities/boption.js"; import BEtapiToken from "./entities/betapi_token.js"; +import BNoteEmbedding from "./entities/bnote_embedding.js"; import cls from "../services/cls.js"; import entityConstructor from "../becca/entity_constructor.js"; -import type { AttributeRow, BranchRow, EtapiTokenRow, NoteRow, OptionRow } from "@triliumnext/commons"; +import type { AttributeRow, BranchRow, EtapiTokenRow, NoteRow, OptionRow, NoteEmbeddingRow } from "@triliumnext/commons"; import type AbstractBeccaEntity from "./entities/abstract_becca_entity.js"; import ws from "../services/ws.js"; @@ -63,6 +64,10 @@ function load() { for (const row of sql.getRows(/*sql*/`SELECT etapiTokenId, name, tokenHash, utcDateCreated, utcDateModified FROM etapi_tokens WHERE isDeleted = 0`)) { new BEtapiToken(row); } + + for (const row of sql.getRows(/*sql*/`SELECT embedId, noteId, providerId, modelId, dimension, embedding, version, dateCreated, dateModified, utcDateCreated, utcDateModified FROM note_embeddings`)) { + new BNoteEmbedding(row).init(); + } }); for (const noteId in becca.notes) { @@ -85,7 +90,7 @@ eventService.subscribeBeccaLoader([eventService.ENTITY_CHANGE_SYNCED], ({ entity return; } - if (["notes", "branches", "attributes", "etapi_tokens", "options"].includes(entityName)) { + if (["notes", "branches", "attributes", "etapi_tokens", "options", "note_embeddings"].includes(entityName)) { const EntityClass = entityConstructor.getEntityFromEntityName(entityName); const primaryKeyName = EntityClass.primaryKeyName; @@ -143,6 +148,8 @@ eventService.subscribeBeccaLoader([eventService.ENTITY_DELETED, eventService.ENT attributeDeleted(entityId); } else if (entityName === "etapi_tokens") { etapiTokenDeleted(entityId); + } else if (entityName === "note_embeddings") { + noteEmbeddingDeleted(entityId); } }); @@ -278,6 +285,10 @@ function etapiTokenDeleted(etapiTokenId: string) { delete becca.etapiTokens[etapiTokenId]; } +function noteEmbeddingDeleted(embedId: string) { + delete becca.noteEmbeddings[embedId]; +} + eventService.subscribeBeccaLoader(eventService.ENTER_PROTECTED_SESSION, () => { try { becca.decryptProtectedNotes(); diff --git a/apps/server/src/becca/entities/bnote_embedding.ts b/apps/server/src/becca/entities/bnote_embedding.ts index 76d559e52..c59a06e5e 100644 --- a/apps/server/src/becca/entities/bnote_embedding.ts +++ b/apps/server/src/becca/entities/bnote_embedding.ts @@ -32,6 +32,12 @@ class BNoteEmbedding extends AbstractBeccaEntity { } } + init() { + if (this.embedId) { + this.becca.noteEmbeddings[this.embedId] = this; + } + } + updateFromRow(row: NoteEmbeddingRow): void { this.embedId = row.embedId; this.noteId = row.noteId; @@ -44,6 +50,10 @@ class BNoteEmbedding extends AbstractBeccaEntity { this.dateModified = row.dateModified; this.utcDateCreated = row.utcDateCreated; this.utcDateModified = row.utcDateModified; + + if (this.embedId) { + this.becca.noteEmbeddings[this.embedId] = this; + } } override beforeSaving() { From cc0795f812fc28429ae90db1ae081c34f1e59f61 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Mon, 2 Jun 2025 15:49:45 +0000 Subject: [PATCH 2/4] fix(llm): also add note_embeddings to froca and consistency tests --- apps/client/src/services/froca_updater.ts | 2 +- apps/client/src/services/load_results.ts | 2 ++ apps/server/src/services/consistency_checks.ts | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/client/src/services/froca_updater.ts b/apps/client/src/services/froca_updater.ts index 1f8eaa541..d01aa7a73 100644 --- a/apps/client/src/services/froca_updater.ts +++ b/apps/client/src/services/froca_updater.ts @@ -35,7 +35,7 @@ async function processEntityChanges(entityChanges: EntityChange[]) { loadResults.addOption(attributeEntity.name); } else if (ec.entityName === "attachments") { processAttachment(loadResults, ec); - } else if (ec.entityName === "blobs" || ec.entityName === "etapi_tokens") { + } else if (ec.entityName === "blobs" || ec.entityName === "etapi_tokens" || ec.entityName === "note_embeddings") { // NOOP } else { throw new Error(`Unknown entityName '${ec.entityName}'`); diff --git a/apps/client/src/services/load_results.ts b/apps/client/src/services/load_results.ts index 11f9a1a11..3f034a1e2 100644 --- a/apps/client/src/services/load_results.ts +++ b/apps/client/src/services/load_results.ts @@ -48,6 +48,7 @@ interface ContentNoteIdToComponentIdRow { noteId: string; componentId: string; } +interface NoteEmbeddingRow {} type EntityRowMappings = { notes: NoteRow; @@ -56,6 +57,7 @@ type EntityRowMappings = { options: OptionRow; revisions: RevisionRow; note_reordering: NoteReorderingRow; + note_embeddings: NoteEmbeddingRow; }; export type EntityRowNames = keyof EntityRowMappings; diff --git a/apps/server/src/services/consistency_checks.ts b/apps/server/src/services/consistency_checks.ts index ec7850572..8022c74df 100644 --- a/apps/server/src/services/consistency_checks.ts +++ b/apps/server/src/services/consistency_checks.ts @@ -799,6 +799,7 @@ class ConsistencyChecks { this.runEntityChangeChecks("attributes", "attributeId"); this.runEntityChangeChecks("etapi_tokens", "etapiTokenId"); this.runEntityChangeChecks("options", "name"); + this.runEntityChangeChecks("note_embeddings", "embedId"); } findWronglyNamedAttributes() { From b0d60f80040081d6a9928a0b78d7d0d3ed287571 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Mon, 2 Jun 2025 19:01:34 +0000 Subject: [PATCH 3/4] refactor(llm): update NoteEmbeddingRow structure and add handling in LoadResults class --- apps/client/src/services/froca_updater.ts | 2 +- apps/client/src/services/load_results.ts | 27 +++++++++++++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/apps/client/src/services/froca_updater.ts b/apps/client/src/services/froca_updater.ts index d01aa7a73..27d25922a 100644 --- a/apps/client/src/services/froca_updater.ts +++ b/apps/client/src/services/froca_updater.ts @@ -36,7 +36,7 @@ async function processEntityChanges(entityChanges: EntityChange[]) { } else if (ec.entityName === "attachments") { processAttachment(loadResults, ec); } else if (ec.entityName === "blobs" || ec.entityName === "etapi_tokens" || ec.entityName === "note_embeddings") { - // NOOP + // NOOP - these entities don't require frontend processing } else { throw new Error(`Unknown entityName '${ec.entityName}'`); } diff --git a/apps/client/src/services/load_results.ts b/apps/client/src/services/load_results.ts index 3f034a1e2..9762463f6 100644 --- a/apps/client/src/services/load_results.ts +++ b/apps/client/src/services/load_results.ts @@ -44,11 +44,18 @@ interface OptionRow {} interface NoteReorderingRow {} -interface ContentNoteIdToComponentIdRow { +interface NoteEmbeddingRow { + embedId: string; noteId: string; - componentId: string; + providerId: string; + modelId: string; + dimension: number; + version: number; + dateCreated: string; + utcDateCreated: string; + dateModified: string; + utcDateModified: string; } -interface NoteEmbeddingRow {} type EntityRowMappings = { notes: NoteRow; @@ -73,6 +80,7 @@ export default class LoadResults { private contentNoteIdToComponentId: ContentNoteIdToComponentIdRow[]; private optionNames: string[]; private attachmentRows: AttachmentRow[]; + private noteEmbeddingRows: NoteEmbeddingRow[]; constructor(entityChanges: EntityChange[]) { const entities: Record> = {}; @@ -101,6 +109,8 @@ export default class LoadResults { this.optionNames = []; this.attachmentRows = []; + + this.noteEmbeddingRows = []; } getEntityRow(entityName: T, entityId: string): EntityRowMappings[T] { @@ -203,6 +213,14 @@ export default class LoadResults { return this.attachmentRows; } + addNoteEmbedding(embedding: NoteEmbeddingRow) { + this.noteEmbeddingRows.push(embedding); + } + + getNoteEmbeddingRows() { + return this.noteEmbeddingRows; + } + /** * @returns {boolean} true if there are changes which could affect the attributes (including inherited ones) * notably changes in note itself should not have any effect on attributes @@ -220,7 +238,8 @@ export default class LoadResults { this.revisionRows.length === 0 && this.contentNoteIdToComponentId.length === 0 && this.optionNames.length === 0 && - this.attachmentRows.length === 0 + this.attachmentRows.length === 0 && + this.noteEmbeddingRows.length === 0 ); } From 934efab533aa40da8c9b6677dfe0f6ba9636cd88 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Mon, 2 Jun 2025 19:10:47 +0000 Subject: [PATCH 4/4] refactor(llm): update handling of note embeddings and clean up LoadResults class --- apps/client/src/services/froca_updater.ts | 2 +- apps/client/src/services/load_results.ts | 14 +------------- apps/server/src/services/ws.ts | 7 +++++++ 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/apps/client/src/services/froca_updater.ts b/apps/client/src/services/froca_updater.ts index 27d25922a..412d8d6cd 100644 --- a/apps/client/src/services/froca_updater.ts +++ b/apps/client/src/services/froca_updater.ts @@ -36,7 +36,7 @@ async function processEntityChanges(entityChanges: EntityChange[]) { } else if (ec.entityName === "attachments") { processAttachment(loadResults, ec); } else if (ec.entityName === "blobs" || ec.entityName === "etapi_tokens" || ec.entityName === "note_embeddings") { - // NOOP - these entities don't require frontend processing + // NOOP - these entities are handled at the backend level and don't require frontend processing } else { throw new Error(`Unknown entityName '${ec.entityName}'`); } diff --git a/apps/client/src/services/load_results.ts b/apps/client/src/services/load_results.ts index 9762463f6..59d201f2b 100644 --- a/apps/client/src/services/load_results.ts +++ b/apps/client/src/services/load_results.ts @@ -80,7 +80,6 @@ export default class LoadResults { private contentNoteIdToComponentId: ContentNoteIdToComponentIdRow[]; private optionNames: string[]; private attachmentRows: AttachmentRow[]; - private noteEmbeddingRows: NoteEmbeddingRow[]; constructor(entityChanges: EntityChange[]) { const entities: Record> = {}; @@ -109,8 +108,6 @@ export default class LoadResults { this.optionNames = []; this.attachmentRows = []; - - this.noteEmbeddingRows = []; } getEntityRow(entityName: T, entityId: string): EntityRowMappings[T] { @@ -213,14 +210,6 @@ export default class LoadResults { return this.attachmentRows; } - addNoteEmbedding(embedding: NoteEmbeddingRow) { - this.noteEmbeddingRows.push(embedding); - } - - getNoteEmbeddingRows() { - return this.noteEmbeddingRows; - } - /** * @returns {boolean} true if there are changes which could affect the attributes (including inherited ones) * notably changes in note itself should not have any effect on attributes @@ -238,8 +227,7 @@ export default class LoadResults { this.revisionRows.length === 0 && this.contentNoteIdToComponentId.length === 0 && this.optionNames.length === 0 && - this.attachmentRows.length === 0 && - this.noteEmbeddingRows.length === 0 + this.attachmentRows.length === 0 ); } diff --git a/apps/server/src/services/ws.ts b/apps/server/src/services/ws.ts index 8dde51639..6a211c572 100644 --- a/apps/server/src/services/ws.ts +++ b/apps/server/src/services/ws.ts @@ -203,6 +203,13 @@ function fillInAdditionalProperties(entityChange: EntityChange) { WHERE attachmentId = ?`, [entityChange.entityId] ); + } else if (entityChange.entityName === "note_embeddings") { + // Note embeddings are backend-only entities for AI/vector search + // Frontend doesn't need the full embedding data (which is large binary data) + // Just ensure entity is marked as handled - actual sync happens at database level + if (!entityChange.isErased) { + entityChange.entity = { embedId: entityChange.entityId }; + } } if (entityChange.entity instanceof AbstractBeccaEntity) {