From 7ae55de8b1c3b8eb3328f13798b01c5fcb32484c Mon Sep 17 00:00:00 2001 From: perf3ct Date: Tue, 1 Apr 2025 18:44:10 +0000 Subject: [PATCH] move the embeddings api endpoint to underneath llm --- .../options/ai_settings/ai_settings_widget.ts | 30 ++-- src/routes/api/embeddings.ts | 156 +++++++++++------- src/routes/routes.ts | 28 ++-- 3 files changed, 124 insertions(+), 90 deletions(-) diff --git a/src/public/app/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts b/src/public/app/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts index 89530a17a..aa5e35f28 100644 --- a/src/public/app/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts +++ b/src/public/app/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts @@ -38,19 +38,19 @@ export default class AiSettingsWidget extends OptionsWidget { */ setupChangeHandler(selector: string, optionName: keyof OptionDefinitions, validateAfter: boolean = false, isCheckbox: boolean = false) { if (!this.$widget) return; - + const $element = this.$widget.find(selector); $element.on('change', async () => { let value: string; - + if (isCheckbox) { value = $element.prop('checked') ? 'true' : 'false'; } else { value = $element.val() as string; } - + await this.updateOption(optionName, value); - + if (validateAfter) { await this.displayValidationWarnings(); } @@ -68,22 +68,22 @@ export default class AiSettingsWidget extends OptionsWidget { this.setupChangeHandler('.ai-provider-precedence', 'aiProviderPrecedence', true); this.setupChangeHandler('.ai-temperature', 'aiTemperature'); this.setupChangeHandler('.ai-system-prompt', 'aiSystemPrompt'); - + // OpenAI options this.setupChangeHandler('.openai-api-key', 'openaiApiKey', true); this.setupChangeHandler('.openai-base-url', 'openaiBaseUrl', true); this.setupChangeHandler('.openai-default-model', 'openaiDefaultModel'); this.setupChangeHandler('.openai-embedding-model', 'openaiEmbeddingModel'); - + // Anthropic options this.setupChangeHandler('.anthropic-api-key', 'anthropicApiKey', true); this.setupChangeHandler('.anthropic-default-model', 'anthropicDefaultModel'); this.setupChangeHandler('.anthropic-base-url', 'anthropicBaseUrl'); - + // Voyage options this.setupChangeHandler('.voyage-api-key', 'voyageApiKey'); this.setupChangeHandler('.voyage-embedding-model', 'voyageEmbeddingModel'); - + // Ollama options this.setupChangeHandler('.ollama-base-url', 'ollamaBaseUrl'); this.setupChangeHandler('.ollama-default-model', 'ollamaDefaultModel'); @@ -150,9 +150,9 @@ export default class AiSettingsWidget extends OptionsWidget { $recreateEmbeddings.on('click', async () => { if (confirm(t("ai_llm.recreate_embeddings_confirm") || "Are you sure you want to recreate all embeddings? This may take a long time.")) { try { - await server.post('embeddings/reprocess'); + await server.post('llm/embeddings/reprocess'); toastService.showMessage(t("ai_llm.recreate_embeddings_started")); - + // Start progress polling this.pollIndexRebuildProgress(); } catch (e) { @@ -161,12 +161,12 @@ export default class AiSettingsWidget extends OptionsWidget { } } }); - + // Rebuild index button const $rebuildIndex = this.$widget.find('.rebuild-embeddings-index'); $rebuildIndex.on('click', async () => { try { - await server.post('embeddings/rebuild-index'); + await server.post('llm/embeddings/rebuild-index'); toastService.showMessage(t("ai_llm.rebuild_index_started")); // Start progress polling @@ -300,7 +300,7 @@ export default class AiSettingsWidget extends OptionsWidget { if (!this.$widget) return; try { - const response = await server.get('embeddings/stats'); + const response = await server.get('llm/embeddings/stats'); if (response && response.success) { const stats = response.stats; @@ -357,7 +357,7 @@ export default class AiSettingsWidget extends OptionsWidget { if (!this.$widget) return; try { - const response = await server.get('embeddings/failed'); + const response = await server.get('llm/embeddings/failed'); if (response && response.success) { const failedNotes = response.failedNotes || []; @@ -412,7 +412,7 @@ export default class AiSettingsWidget extends OptionsWidget { $failedNotesList.find('.retry-embedding').on('click', async function() { const noteId = $(this).closest('tr').data('note-id'); try { - await server.post('embeddings/retry', { noteId }); + await server.post('llm/embeddings/retry', { noteId }); toastService.showMessage(t("ai_llm.retry_queued")); // Remove this row or update status $(this).closest('tr').remove(); diff --git a/src/routes/api/embeddings.ts b/src/routes/api/embeddings.ts index f35287a64..012a9c82f 100644 --- a/src/routes/api/embeddings.ts +++ b/src/routes/api/embeddings.ts @@ -9,7 +9,7 @@ import sql from "../../services/sql.js"; /** * @swagger - * /api/embeddings/similar/{noteId}: + * /api/llm/embeddings/similar/{noteId}: * get: * summary: Find similar notes based on a given note ID * operationId: embeddings-similar-by-note @@ -139,7 +139,7 @@ async function findSimilarNotes(req: Request, res: Response) { /** * @swagger - * /api/embeddings/search: + * /api/llm/embeddings/search: * post: * summary: Search for notes similar to provided text * operationId: embeddings-search-by-text @@ -250,7 +250,7 @@ async function searchByText(req: Request, res: Response) { /** * @swagger - * /api/embeddings/providers: + * /api/llm/embeddings/providers: * get: * summary: Get available embedding providers * operationId: embeddings-get-providers @@ -294,7 +294,7 @@ async function getProviders(req: Request, res: Response) { /** * @swagger - * /api/embeddings/providers/{providerId}: + * /api/llm/embeddings/providers/{providerId}: * patch: * summary: Update embedding provider configuration * operationId: embeddings-update-provider @@ -304,7 +304,7 @@ async function getProviders(req: Request, res: Response) { * required: true * schema: * type: string - * description: ID of the embedding provider to update + * description: Provider ID to update * requestBody: * required: true * content: @@ -312,18 +312,18 @@ async function getProviders(req: Request, res: Response) { * schema: * type: object * properties: - * isEnabled: + * enabled: * type: boolean - * description: Whether the provider is enabled + * description: Whether provider is enabled * priority: * type: integer - * description: Priority level for the provider + * description: Priority order (lower is higher priority) * config: * type: object * description: Provider-specific configuration * responses: * '200': - * description: Provider successfully updated + * description: Provider updated successfully * content: * application/json: * schema: @@ -331,8 +331,8 @@ async function getProviders(req: Request, res: Response) { * properties: * success: * type: boolean - * '404': - * description: Provider not found + * '400': + * description: Invalid provider ID or configuration * security: * - session: [] * tags: ["llm"] @@ -359,13 +359,29 @@ async function updateProvider(req: Request, res: Response) { /** * @swagger - * /api/embeddings/reprocess: + * /api/llm/embeddings/reprocess: * post: - * summary: Reprocess all notes for embedding generation + * summary: Reprocess embeddings for all notes * operationId: embeddings-reprocess-all + * requestBody: + * required: true + * content: + * application/json: + * schema: + * type: object + * properties: + * providerId: + * type: string + * description: Provider ID to use for reprocessing + * modelId: + * type: string + * description: Model ID to use for reprocessing + * forceReprocess: + * type: boolean + * description: Whether to reprocess notes that already have embeddings * responses: * '200': - * description: Reprocessing started successfully + * description: Reprocessing started * content: * application/json: * schema: @@ -373,8 +389,12 @@ async function updateProvider(req: Request, res: Response) { * properties: * success: * type: boolean + * jobId: + * type: string * message: * type: string + * '400': + * description: Invalid provider ID or configuration * security: * - session: [] * tags: ["llm"] @@ -405,10 +425,17 @@ async function reprocessAllNotes(req: Request, res: Response) { /** * @swagger - * /api/embeddings/queue-status: + * /api/llm/embeddings/queue-status: * get: - * summary: Get status of the embedding generation queue + * summary: Get status of the embedding processing queue * operationId: embeddings-queue-status + * parameters: + * - name: jobId + * in: query + * required: false + * schema: + * type: string + * description: Optional job ID to get status for a specific processing job * responses: * '200': * description: Queue status information @@ -420,17 +447,14 @@ async function reprocessAllNotes(req: Request, res: Response) { * success: * type: boolean * status: + * type: string + * enum: [idle, processing, paused] + * progress: + * type: number + * format: float + * description: Progress percentage (0-100) + * details: * type: object - * properties: - * queueCount: - * type: integer - * description: Number of items in the queue - * failedCount: - * type: integer - * description: Number of failed embedding attempts - * totalEmbeddingsCount: - * type: integer - * description: Total number of generated embeddings * security: * - session: [] * tags: ["llm"] @@ -461,7 +485,7 @@ async function getQueueStatus(req: Request, res: Response) { /** * @swagger - * /api/embeddings/stats: + * /api/llm/embeddings/stats: * get: * summary: Get embedding statistics * operationId: embeddings-stats @@ -502,13 +526,13 @@ async function getEmbeddingStats(req: Request, res: Response) { /** * @swagger - * /api/embeddings/failed: + * /api/llm/embeddings/failed: * get: * summary: Get list of notes that failed embedding generation * operationId: embeddings-failed-notes * responses: * '200': - * description: List of failed embedding notes + * description: List of failed notes * content: * application/json: * schema: @@ -527,9 +551,7 @@ async function getEmbeddingStats(req: Request, res: Response) { * type: string * error: * type: string - * attempts: - * type: integer - * lastAttempt: + * failedAt: * type: string * format: date-time * security: @@ -549,9 +571,9 @@ async function getFailedNotes(req: Request, res: Response) { /** * @swagger - * /api/embeddings/retry/{noteId}: + * /api/llm/embeddings/retry/{noteId}: * post: - * summary: Retry embedding generation for a failed note + * summary: Retry generating embeddings for a failed note * operationId: embeddings-retry-note * parameters: * - name: noteId @@ -559,10 +581,22 @@ async function getFailedNotes(req: Request, res: Response) { * required: true * schema: * type: string - * description: ID of the note to retry embedding + * description: Note ID to retry + * - name: providerId + * in: query + * required: false + * schema: + * type: string + * description: Provider ID to use (defaults to configured default) + * - name: modelId + * in: query + * required: false + * schema: + * type: string + * description: Model ID to use (defaults to provider default) * responses: * '200': - * description: Retry operation result + * description: Retry result * content: * application/json: * schema: @@ -572,8 +606,10 @@ async function getFailedNotes(req: Request, res: Response) { * type: boolean * message: * type: string + * '400': + * description: Invalid request * '404': - * description: Note not found or not in failed state + * description: Note not found * security: * - session: [] * tags: ["llm"] @@ -605,13 +641,26 @@ async function retryFailedNote(req: Request, res: Response) { /** * @swagger - * /api/embeddings/retry-all-failed: + * /api/llm/embeddings/retry-all-failed: * post: - * summary: Retry embedding generation for all failed notes + * summary: Retry generating embeddings for all failed notes * operationId: embeddings-retry-all-failed + * requestBody: + * required: false + * content: + * application/json: + * schema: + * type: object + * properties: + * providerId: + * type: string + * description: Provider ID to use (defaults to configured default) + * modelId: + * type: string + * description: Model ID to use (defaults to provider default) * responses: * '200': - * description: Retry operation started + * description: Retry started * content: * application/json: * schema: @@ -621,9 +670,8 @@ async function retryFailedNote(req: Request, res: Response) { * type: boolean * message: * type: string - * count: - * type: integer - * description: Number of notes queued for retry + * jobId: + * type: string * security: * - session: [] * tags: ["llm"] @@ -639,26 +687,13 @@ async function retryAllFailedNotes(req: Request, res: Response) { /** * @swagger - * /api/embeddings/rebuild-index: + * /api/llm/embeddings/rebuild-index: * post: - * summary: Rebuild the embedding vector index + * summary: Rebuild the vector store index * operationId: embeddings-rebuild-index - * requestBody: - * required: false - * content: - * application/json: - * schema: - * type: object - * properties: - * provider: - * type: string - * description: Specific provider to rebuild index for - * force: - * type: boolean - * description: Force rebuild even if not necessary * responses: * '200': - * description: Index rebuild operation started + * description: Rebuild started * content: * application/json: * schema: @@ -670,7 +705,6 @@ async function retryAllFailedNotes(req: Request, res: Response) { * type: string * jobId: * type: string - * description: ID of the rebuild job for status tracking * security: * - session: [] * tags: ["llm"] @@ -695,7 +729,7 @@ async function rebuildIndex(req: Request, res: Response) { /** * @swagger - * /api/embeddings/index-rebuild-status: + * /api/llm/embeddings/index-rebuild-status: * get: * summary: Get status of the vector index rebuild operation * operationId: embeddings-rebuild-status diff --git a/src/routes/routes.ts b/src/routes/routes.ts index 0b3ef9ff5..9c95580f8 100644 --- a/src/routes/routes.ts +++ b/src/routes/routes.ts @@ -391,20 +391,6 @@ function register(app: express.Application) { etapiSpecRoute.register(router); etapiBackupRoute.register(router); - // Embeddings API endpoints - apiRoute(GET, "/api/embeddings/similar/:noteId", embeddingsRoute.findSimilarNotes); - apiRoute(PST, "/api/embeddings/search", embeddingsRoute.searchByText); - apiRoute(GET, "/api/embeddings/providers", embeddingsRoute.getProviders); - apiRoute(PATCH, "/api/embeddings/providers/:providerId", embeddingsRoute.updateProvider); - apiRoute(PST, "/api/embeddings/reprocess", embeddingsRoute.reprocessAllNotes); - apiRoute(GET, "/api/embeddings/queue-status", embeddingsRoute.getQueueStatus); - apiRoute(GET, "/api/embeddings/stats", embeddingsRoute.getEmbeddingStats); - apiRoute(GET, "/api/embeddings/failed", embeddingsRoute.getFailedNotes); - apiRoute(PST, "/api/embeddings/retry/:noteId", embeddingsRoute.retryFailedNote); - apiRoute(PST, "/api/embeddings/retry-all-failed", embeddingsRoute.retryAllFailedNotes); - apiRoute(PST, "/api/embeddings/rebuild-index", embeddingsRoute.rebuildIndex); - apiRoute(GET, "/api/embeddings/index-rebuild-status", embeddingsRoute.getIndexRebuildStatus); - // LLM chat session management endpoints apiRoute(PST, "/api/llm/sessions", llmRoute.createSession); apiRoute(GET, "/api/llm/sessions", llmRoute.listSessions); @@ -424,6 +410,20 @@ function register(app: express.Application) { apiRoute(GET, "/api/llm/indexes/context", llmRoute.generateQueryContext); // Get context apiRoute(PST, "/api/llm/indexes/notes/:noteId", llmRoute.indexNote); // Create index for specific note + // LLM embeddings endpoints + apiRoute(GET, "/api/llm/embeddings/similar/:noteId", embeddingsRoute.findSimilarNotes); + apiRoute(PST, "/api/llm/embeddings/search", embeddingsRoute.searchByText); + apiRoute(GET, "/api/llm/embeddings/providers", embeddingsRoute.getProviders); + apiRoute(PATCH, "/api/llm/embeddings/providers/:providerId", embeddingsRoute.updateProvider); + apiRoute(PST, "/api/llm/embeddings/reprocess", embeddingsRoute.reprocessAllNotes); + apiRoute(GET, "/api/llm/embeddings/queue-status", embeddingsRoute.getQueueStatus); + apiRoute(GET, "/api/llm/embeddings/stats", embeddingsRoute.getEmbeddingStats); + apiRoute(GET, "/api/llm/embeddings/failed", embeddingsRoute.getFailedNotes); + apiRoute(PST, "/api/llm/embeddings/retry/:noteId", embeddingsRoute.retryFailedNote); + apiRoute(PST, "/api/llm/embeddings/retry-all-failed", embeddingsRoute.retryAllFailedNotes); + apiRoute(PST, "/api/llm/embeddings/rebuild-index", embeddingsRoute.rebuildIndex); + apiRoute(GET, "/api/llm/embeddings/index-rebuild-status", embeddingsRoute.getIndexRebuildStatus); + // LLM provider endpoints - moved under /api/llm/providers hierarchy apiRoute(GET, "/api/llm/providers/ollama/models", ollamaRoute.listModels); apiRoute(GET, "/api/llm/providers/openai/models", openaiRoute.listModels);