diff --git a/db/migrations/0230__vector_embeddings.sql b/db/migrations/0230__vector_embeddings.sql index e3e8bc217..a614948a1 100644 --- a/db/migrations/0230__vector_embeddings.sql +++ b/db/migrations/0230__vector_embeddings.sql @@ -1,4 +1,5 @@ -- Add tables for vector embeddings storage and management +-- This migration adds embedding support to the main document.db database -- Store embeddings for notes CREATE TABLE IF NOT EXISTS "note_embeddings" ( @@ -44,7 +45,11 @@ CREATE TABLE IF NOT EXISTS "embedding_providers" ( ); -- Add default embedding provider options -INSERT INTO options (name, value, isSynced) VALUES ('embeddingAutoUpdateEnabled', 'true', 1); -INSERT INTO options (name, value, isSynced) VALUES ('embeddingUpdateInterval', '5000', 1); -- 5 seconds -INSERT INTO options (name, value, isSynced) VALUES ('embeddingBatchSize', '10', 1); -INSERT INTO options (name, value, isSynced) VALUES ('embeddingDefaultDimension', '1536', 1); \ No newline at end of file +INSERT INTO options (name, value, isSynced, utcDateModified) +VALUES ('embeddingAutoUpdateEnabled', 'true', 1, strftime('%Y-%m-%d %H:%M:%f', 'now')); +INSERT INTO options (name, value, isSynced, utcDateModified) +VALUES ('embeddingUpdateInterval', '5000', 1, strftime('%Y-%m-%d %H:%M:%f', 'now')); -- 5 seconds +INSERT INTO options (name, value, isSynced, utcDateModified) +VALUES ('embeddingBatchSize', '10', 1, strftime('%Y-%m-%d %H:%M:%f', 'now')); +INSERT INTO options (name, value, isSynced, utcDateModified) +VALUES ('embeddingDefaultDimension', '1536', 1, strftime('%Y-%m-%d %H:%M:%f', 'now')); \ No newline at end of file diff --git a/db/schema.sql b/db/schema.sql index 1b4c46321..a4f153042 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -132,3 +132,43 @@ CREATE INDEX IDX_attachments_ownerId_role CREATE INDEX IDX_notes_blobId on notes (blobId); CREATE INDEX IDX_revisions_blobId on revisions (blobId); CREATE INDEX IDX_attachments_blobId on attachments (blobId); + +CREATE TABLE IF NOT EXISTS "note_embeddings" ( + "embedId" TEXT NOT NULL PRIMARY KEY, + "noteId" TEXT NOT NULL, + "providerId" TEXT NOT NULL, + "modelId" TEXT NOT NULL, + "dimension" INTEGER NOT NULL, + "embedding" BLOB NOT NULL, + "version" INTEGER NOT NULL DEFAULT 1, + "dateCreated" TEXT NOT NULL, + "utcDateCreated" TEXT NOT NULL, + "dateModified" TEXT NOT NULL, + "utcDateModified" TEXT NOT NULL +); + +CREATE INDEX "IDX_note_embeddings_noteId" ON "note_embeddings" ("noteId"); +CREATE INDEX "IDX_note_embeddings_providerId_modelId" ON "note_embeddings" ("providerId", "modelId"); + +CREATE TABLE IF NOT EXISTS "embedding_queue" ( + "noteId" TEXT NOT NULL PRIMARY KEY, + "operation" TEXT NOT NULL, + "dateQueued" TEXT NOT NULL, + "utcDateQueued" TEXT NOT NULL, + "priority" INTEGER NOT NULL DEFAULT 0, + "attempts" INTEGER NOT NULL DEFAULT 0, + "lastAttempt" TEXT NULL, + "error" TEXT NULL +); + +CREATE TABLE IF NOT EXISTS "embedding_providers" ( + "providerId" TEXT NOT NULL PRIMARY KEY, + "name" TEXT NOT NULL, + "isEnabled" INTEGER NOT NULL DEFAULT 0, + "priority" INTEGER NOT NULL DEFAULT 0, + "config" TEXT NOT NULL, + "dateCreated" TEXT NOT NULL, + "utcDateCreated" TEXT NOT NULL, + "dateModified" TEXT NOT NULL, + "utcDateModified" TEXT NOT NULL +); diff --git a/src/routes/api/options.ts b/src/routes/api/options.ts index 394629ccb..21903b09e 100644 --- a/src/routes/api/options.ts +++ b/src/routes/api/options.ts @@ -90,9 +90,16 @@ const ALLOWED_OPTIONS = new Set([ "ollamaEnabled", "ollamaBaseUrl", "ollamaDefaultModel", + "ollamaEmbeddingModel", "aiProviderPrecedence", "aiTemperature", - "aiSystemPrompt" + "aiSystemPrompt", + // Embedding options + "embeddingAutoUpdate", + "embeddingAutoUpdateEnabled", + "embeddingBatchSize", + "embeddingUpdateInterval", + "embeddingDefaultDimension" ]); function getOptions() {