From a20e36f4ee36365cdd5d3a28ffa2e04d645060ce Mon Sep 17 00:00:00 2001 From: perf3ct Date: Wed, 4 Jun 2025 20:13:13 +0000 Subject: [PATCH 01/18] feat(llm): change from using precedence list to using a sing specified provider for either chat and/or embeddings --- .../options/ai_settings/ai_settings_widget.ts | 113 +++--- .../options/ai_settings/template.ts | 353 ++++++++++-------- apps/server/src/routes/api/options.ts | 4 +- .../src/services/llm/ai_service_manager.ts | 104 +++++- .../llm/config/configuration_helpers.ts | 123 +++--- .../llm/config/configuration_manager.ts | 81 ++-- .../llm/context/modules/provider_manager.ts | 43 +-- apps/server/src/services/llm/index_service.ts | 40 +- .../interfaces/configuration_interfaces.ts | 6 +- .../pipeline/stages/model_selection_stage.ts | 186 +++++++-- .../llm/providers/anthropic_service.ts | 8 + .../services/llm/providers/ollama_service.ts | 9 + .../services/llm/providers/openai_service.ts | 8 + apps/server/src/services/options_init.ts | 16 +- packages/commons/src/lib/options_interface.ts | 5 +- 15 files changed, 685 insertions(+), 414 deletions(-) diff --git a/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts b/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts index da7321d18..30f7d3408 100644 --- a/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts +++ b/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts @@ -65,7 +65,7 @@ export default class AiSettingsWidget extends OptionsWidget { // Core AI options this.setupChangeHandler('.ai-enabled', 'aiEnabled', true, true); - this.setupChangeHandler('.ai-provider-precedence', 'aiProviderPrecedence', true); + this.setupChangeHandler('.ai-selected-provider', 'aiSelectedProvider', true); this.setupChangeHandler('.ai-temperature', 'aiTemperature'); this.setupChangeHandler('.ai-system-prompt', 'aiSystemPrompt'); @@ -132,11 +132,28 @@ export default class AiSettingsWidget extends OptionsWidget { this.setupChangeHandler('.enable-automatic-indexing', 'enableAutomaticIndexing', false, true); this.setupChangeHandler('.embedding-similarity-threshold', 'embeddingSimilarityThreshold'); this.setupChangeHandler('.max-notes-per-llm-query', 'maxNotesPerLlmQuery'); - this.setupChangeHandler('.embedding-provider-precedence', 'embeddingProviderPrecedence', true); + this.setupChangeHandler('.embedding-selected-provider', 'embeddingSelectedProvider', true); this.setupChangeHandler('.embedding-dimension-strategy', 'embeddingDimensionStrategy'); this.setupChangeHandler('.embedding-batch-size', 'embeddingBatchSize'); this.setupChangeHandler('.embedding-update-interval', 'embeddingUpdateInterval'); + // Add provider selection change handlers for dynamic settings visibility + this.$widget.find('.ai-selected-provider').on('change', () => { + const selectedProvider = this.$widget.find('.ai-selected-provider').val() as string; + this.$widget.find('.provider-settings').hide(); + if (selectedProvider) { + this.$widget.find(`.${selectedProvider}-provider-settings`).show(); + } + }); + + this.$widget.find('.embedding-selected-provider').on('change', () => { + const selectedProvider = this.$widget.find('.embedding-selected-provider').val() as string; + this.$widget.find('.embedding-provider-settings').hide(); + if (selectedProvider) { + this.$widget.find(`.${selectedProvider}-embedding-provider-settings`).show(); + } + }); + // No sortable behavior needed anymore // Embedding stats refresh button @@ -194,42 +211,25 @@ export default class AiSettingsWidget extends OptionsWidget { return; } - // Get provider precedence - const providerPrecedence = (this.$widget.find('.ai-provider-precedence').val() as string || '').split(','); + // Get selected provider + const selectedProvider = this.$widget.find('.ai-selected-provider').val() as string; - // Check for OpenAI configuration if it's in the precedence list - const openaiWarnings: string[] = []; - if (providerPrecedence.includes('openai')) { + // Check for selected provider configuration + const providerWarnings: string[] = []; + if (selectedProvider === 'openai') { const openaiApiKey = this.$widget.find('.openai-api-key').val(); if (!openaiApiKey) { - openaiWarnings.push(t("ai_llm.empty_key_warning.openai")); + providerWarnings.push(t("ai_llm.empty_key_warning.openai")); } - } - - // Check for Anthropic configuration if it's in the precedence list - const anthropicWarnings: string[] = []; - if (providerPrecedence.includes('anthropic')) { + } else if (selectedProvider === 'anthropic') { const anthropicApiKey = this.$widget.find('.anthropic-api-key').val(); if (!anthropicApiKey) { - anthropicWarnings.push(t("ai_llm.empty_key_warning.anthropic")); + providerWarnings.push(t("ai_llm.empty_key_warning.anthropic")); } - } - - // Check for Voyage configuration if it's in the precedence list - const voyageWarnings: string[] = []; - if (providerPrecedence.includes('voyage')) { - const voyageApiKey = this.$widget.find('.voyage-api-key').val(); - if (!voyageApiKey) { - voyageWarnings.push(t("ai_llm.empty_key_warning.voyage")); - } - } - - // Check for Ollama configuration if it's in the precedence list - const ollamaWarnings: string[] = []; - if (providerPrecedence.includes('ollama')) { + } else if (selectedProvider === 'ollama') { const ollamaBaseUrl = this.$widget.find('.ollama-base-url').val(); if (!ollamaBaseUrl) { - ollamaWarnings.push(t("ai_llm.ollama_no_url")); + providerWarnings.push(t("ai_llm.ollama_no_url")); } } @@ -238,27 +238,24 @@ export default class AiSettingsWidget extends OptionsWidget { const embeddingsEnabled = this.$widget.find('.enable-automatic-indexing').prop('checked'); if (embeddingsEnabled) { - const embeddingProviderPrecedence = (this.$widget.find('.embedding-provider-precedence').val() as string || '').split(','); + const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; - if (embeddingProviderPrecedence.includes('openai') && !this.$widget.find('.openai-api-key').val()) { + if (selectedEmbeddingProvider === 'openai' && !this.$widget.find('.openai-api-key').val()) { embeddingWarnings.push(t("ai_llm.empty_key_warning.openai")); } - if (embeddingProviderPrecedence.includes('voyage') && !this.$widget.find('.voyage-api-key').val()) { + if (selectedEmbeddingProvider === 'voyage' && !this.$widget.find('.voyage-api-key').val()) { embeddingWarnings.push(t("ai_llm.empty_key_warning.voyage")); } - if (embeddingProviderPrecedence.includes('ollama') && !this.$widget.find('.ollama-base-url').val()) { + if (selectedEmbeddingProvider === 'ollama' && !this.$widget.find('.ollama-base-url').val()) { embeddingWarnings.push(t("ai_llm.empty_key_warning.ollama")); } } // Combine all warnings const allWarnings = [ - ...openaiWarnings, - ...anthropicWarnings, - ...voyageWarnings, - ...ollamaWarnings, + ...providerWarnings, ...embeddingWarnings ]; @@ -449,6 +446,27 @@ export default class AiSettingsWidget extends OptionsWidget { } } + /** + * Update provider settings visibility based on selected providers + */ + updateProviderSettingsVisibility() { + if (!this.$widget) return; + + // Update AI provider settings visibility + const selectedAiProvider = this.$widget.find('.ai-selected-provider').val() as string; + this.$widget.find('.provider-settings').hide(); + if (selectedAiProvider) { + this.$widget.find(`.${selectedAiProvider}-provider-settings`).show(); + } + + // Update embedding provider settings visibility + const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; + this.$widget.find('.embedding-provider-settings').hide(); + if (selectedEmbeddingProvider) { + this.$widget.find(`.${selectedEmbeddingProvider}-embedding-provider-settings`).show(); + } + } + /** * Called when the options have been loaded from the server */ @@ -459,30 +477,30 @@ export default class AiSettingsWidget extends OptionsWidget { this.$widget.find('.ai-enabled').prop('checked', options.aiEnabled !== 'false'); this.$widget.find('.ai-temperature').val(options.aiTemperature || '0.7'); this.$widget.find('.ai-system-prompt').val(options.aiSystemPrompt || ''); - this.$widget.find('.ai-provider-precedence').val(options.aiProviderPrecedence || 'openai,anthropic,ollama'); + this.$widget.find('.ai-selected-provider').val(options.aiSelectedProvider || 'openai'); // OpenAI Section this.$widget.find('.openai-api-key').val(options.openaiApiKey || ''); - this.$widget.find('.openai-base-url').val(options.openaiBaseUrl || 'https://api.openai_llm.com/v1'); - this.$widget.find('.openai-default-model').val(options.openaiDefaultModel || 'gpt-4o'); - this.$widget.find('.openai-embedding-model').val(options.openaiEmbeddingModel || 'text-embedding-3-small'); + this.$widget.find('.openai-base-url').val(options.openaiBaseUrl || 'https://api.openai.com/v1'); + this.$widget.find('.openai-default-model').val(options.openaiDefaultModel || ''); + this.$widget.find('.openai-embedding-model').val(options.openaiEmbeddingModel || ''); // Anthropic Section this.$widget.find('.anthropic-api-key').val(options.anthropicApiKey || ''); this.$widget.find('.anthropic-base-url').val(options.anthropicBaseUrl || 'https://api.anthropic.com'); - this.$widget.find('.anthropic-default-model').val(options.anthropicDefaultModel || 'claude-3-opus-20240229'); + this.$widget.find('.anthropic-default-model').val(options.anthropicDefaultModel || ''); // Voyage Section this.$widget.find('.voyage-api-key').val(options.voyageApiKey || ''); - this.$widget.find('.voyage-embedding-model').val(options.voyageEmbeddingModel || 'voyage-2'); + this.$widget.find('.voyage-embedding-model').val(options.voyageEmbeddingModel || ''); // Ollama Section this.$widget.find('.ollama-base-url').val(options.ollamaBaseUrl || 'http://localhost:11434'); - this.$widget.find('.ollama-default-model').val(options.ollamaDefaultModel || 'llama3'); - this.$widget.find('.ollama-embedding-model').val(options.ollamaEmbeddingModel || 'nomic-embed-text'); + this.$widget.find('.ollama-default-model').val(options.ollamaDefaultModel || ''); + this.$widget.find('.ollama-embedding-model').val(options.ollamaEmbeddingModel || ''); // Embedding Options - this.$widget.find('.embedding-provider-precedence').val(options.embeddingProviderPrecedence || 'openai,voyage,ollama,local'); + this.$widget.find('.embedding-selected-provider').val(options.embeddingSelectedProvider || 'openai'); this.$widget.find('.embedding-auto-update-enabled').prop('checked', options.embeddingAutoUpdateEnabled !== 'false'); this.$widget.find('.enable-automatic-indexing').prop('checked', options.enableAutomaticIndexing !== 'false'); this.$widget.find('.embedding-similarity-threshold').val(options.embeddingSimilarityThreshold || '0.75'); @@ -491,6 +509,9 @@ export default class AiSettingsWidget extends OptionsWidget { this.$widget.find('.embedding-batch-size').val(options.embeddingBatchSize || '10'); this.$widget.find('.embedding-update-interval').val(options.embeddingUpdateInterval || '5000'); + // Show/hide provider settings based on selected providers + this.updateProviderSettingsVisibility(); + // Display validation warnings this.displayValidationWarnings(); } diff --git a/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts b/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts index f8cd79c81..af00d4474 100644 --- a/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts +++ b/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts @@ -61,9 +61,125 @@ export const TPL = `

${t("ai_llm.provider_configuration")}

- - -
${t("ai_llm.provider_precedence_description")}
+ + +
${t("ai_llm.selected_provider_description")}
+
+ + + + + + + + +
@@ -79,155 +195,98 @@ export const TPL = `
- -
-
- - - - -
-

${t("ai_llm.embeddings_configuration")}

- - -
${t("ai_llm.embedding_provider_precedence_description")}
+ + +
${t("ai_llm.selected_embedding_provider_description")}
+
+ + + + + + + + + + + +
diff --git a/apps/server/src/routes/api/options.ts b/apps/server/src/routes/api/options.ts index c69f7568f..ce6275c8e 100644 --- a/apps/server/src/routes/api/options.ts +++ b/apps/server/src/routes/api/options.ts @@ -95,7 +95,7 @@ const ALLOWED_OPTIONS = new Set([ "aiEnabled", "aiTemperature", "aiSystemPrompt", - "aiProviderPrecedence", + "aiSelectedProvider", "openaiApiKey", "openaiBaseUrl", "openaiDefaultModel", @@ -110,7 +110,7 @@ const ALLOWED_OPTIONS = new Set([ "ollamaEmbeddingModel", "embeddingAutoUpdateEnabled", "embeddingDimensionStrategy", - "embeddingProviderPrecedence", + "embeddingSelectedProvider", "embeddingSimilarityThreshold", "embeddingBatchSize", "embeddingUpdateInterval", diff --git a/apps/server/src/services/llm/ai_service_manager.ts b/apps/server/src/services/llm/ai_service_manager.ts index d7bbf4cf7..aeb6e9763 100644 --- a/apps/server/src/services/llm/ai_service_manager.ts +++ b/apps/server/src/services/llm/ai_service_manager.ts @@ -1,4 +1,5 @@ import options from '../options.js'; +import eventService from '../events.js'; import type { AIService, ChatCompletionOptions, ChatResponse, Message } from './ai_interface.js'; import { AnthropicService } from './providers/anthropic_service.js'; import { ContextExtractor } from './context/index.js'; @@ -20,9 +21,8 @@ import type { NoteSearchResult } from './interfaces/context_interfaces.js'; // Import new configuration system import { - getProviderPrecedence, - getPreferredProvider, - getEmbeddingProviderPrecedence, + getSelectedProvider, + getSelectedEmbeddingProvider, parseModelIdentifier, isAIEnabled, getDefaultModelForProvider, @@ -60,6 +60,9 @@ export class AIServiceManager implements IAIServiceManager { this.initializeTools().catch(error => { log.error(`Error initializing LLM tools during AIServiceManager construction: ${error.message || String(error)}`); }); + + // Set up event listener for provider changes + this.setupProviderChangeListener(); } /** @@ -84,16 +87,21 @@ export class AIServiceManager implements IAIServiceManager { } /** - * Update the provider precedence order using the new configuration system + * Update the provider order using the new configuration system (single provider) */ async updateProviderOrderAsync(): Promise { try { - const providers = await getProviderPrecedence(); - this.providerOrder = providers as ServiceProviders[]; + const selectedProvider = await getSelectedProvider(); + if (selectedProvider) { + this.providerOrder = [selectedProvider as ServiceProviders]; + log.info(`Updated provider order: ${selectedProvider}`); + } else { + this.providerOrder = []; + log.info('No provider selected'); + } this.initialized = true; - log.info(`Updated provider order: ${providers.join(', ')}`); } catch (error) { - log.error(`Failed to get provider precedence: ${error}`); + log.error(`Failed to get selected provider: ${error}`); // Keep empty order, will be handled gracefully by other methods this.providerOrder = []; this.initialized = true; @@ -521,13 +529,13 @@ export class AIServiceManager implements IAIServiceManager { */ async getPreferredProviderAsync(): Promise { try { - const preferredProvider = await getPreferredProvider(); - if (preferredProvider === null) { - // No providers configured, fallback to first available - log.info('No providers configured in precedence, using first available provider'); + const selectedProvider = await getSelectedProvider(); + if (selectedProvider === null) { + // No provider selected, fallback to first available + log.info('No provider selected, using first available provider'); return this.providerOrder[0]; } - return preferredProvider; + return selectedProvider; } catch (error) { log.error(`Error getting preferred provider: ${error}`); return this.providerOrder[0]; @@ -580,6 +588,7 @@ export class AIServiceManager implements IAIServiceManager { }; } + /** * Error handler that properly types the error object */ @@ -589,6 +598,75 @@ export class AIServiceManager implements IAIServiceManager { } return String(error); } + + /** + * Set up event listener for provider changes + */ + private setupProviderChangeListener(): void { + // List of AI-related options that should trigger service recreation + const aiRelatedOptions = [ + 'aiSelectedProvider', + 'embeddingSelectedProvider', + 'openaiApiKey', + 'openaiBaseUrl', + 'openaiDefaultModel', + 'anthropicApiKey', + 'anthropicBaseUrl', + 'anthropicDefaultModel', + 'ollamaBaseUrl', + 'ollamaDefaultModel', + 'voyageApiKey' + ]; + + eventService.subscribe(['entityChanged'], ({ entityName, entity }) => { + if (entityName === 'options' && entity && aiRelatedOptions.includes(entity.name)) { + log.info(`AI-related option '${entity.name}' changed, recreating LLM services`); + this.recreateServices(); + } + }); + } + + /** + * Recreate LLM services when provider settings change + */ + private async recreateServices(): Promise { + try { + log.info('Recreating LLM services due to configuration change'); + + // Clear configuration cache first + clearConfigurationCache(); + + // Recreate all service instances to pick up new configuration + this.recreateServiceInstances(); + + // Update provider order with new configuration + await this.updateProviderOrderAsync(); + + log.info('LLM services recreated successfully'); + } catch (error) { + log.error(`Error recreating LLM services: ${this.handleError(error)}`); + } + } + + /** + * Recreate service instances to pick up new configuration + */ + private recreateServiceInstances(): void { + try { + log.info('Recreating service instances'); + + // Recreate service instances + this.services = { + openai: new OpenAIService(), + anthropic: new AnthropicService(), + ollama: new OllamaService() + }; + + log.info('Service instances recreated successfully'); + } catch (error) { + log.error(`Error recreating service instances: ${this.handleError(error)}`); + } + } } // Don't create singleton immediately, use a lazy-loading pattern diff --git a/apps/server/src/services/llm/config/configuration_helpers.ts b/apps/server/src/services/llm/config/configuration_helpers.ts index 88d2cf1da..286716d3c 100644 --- a/apps/server/src/services/llm/config/configuration_helpers.ts +++ b/apps/server/src/services/llm/config/configuration_helpers.ts @@ -1,10 +1,9 @@ import configurationManager from './configuration_manager.js'; +import optionService from '../../options.js'; import type { ProviderType, ModelIdentifier, ModelConfig, - ProviderPrecedenceConfig, - EmbeddingProviderPrecedenceConfig } from '../interfaces/configuration_interfaces.js'; /** @@ -13,41 +12,19 @@ import type { */ /** - * Get the ordered list of AI providers + * Get the selected AI provider */ -export async function getProviderPrecedence(): Promise { - const config = await configurationManager.getProviderPrecedence(); - return config.providers; +export async function getSelectedProvider(): Promise { + const providerOption = optionService.getOption('aiSelectedProvider'); + return providerOption as ProviderType || null; } /** - * Get the default/preferred AI provider + * Get the selected embedding provider */ -export async function getPreferredProvider(): Promise { - const config = await configurationManager.getProviderPrecedence(); - if (config.providers.length === 0) { - return null; // No providers configured - } - return config.defaultProvider || config.providers[0]; -} - -/** - * Get the ordered list of embedding providers - */ -export async function getEmbeddingProviderPrecedence(): Promise { - const config = await configurationManager.getEmbeddingProviderPrecedence(); - return config.providers; -} - -/** - * Get the default embedding provider - */ -export async function getPreferredEmbeddingProvider(): Promise { - const config = await configurationManager.getEmbeddingProviderPrecedence(); - if (config.providers.length === 0) { - return null; // No providers configured - } - return config.defaultProvider || config.providers[0]; +export async function getSelectedEmbeddingProvider(): Promise { + const providerOption = optionService.getOption('embeddingSelectedProvider'); + return providerOption || null; } /** @@ -107,22 +84,20 @@ export async function isProviderConfigured(provider: ProviderType): Promise { - const providers = await getProviderPrecedence(); - - if (providers.length === 0) { - return null; // No providers configured +export async function getAvailableSelectedProvider(): Promise { + const selectedProvider = await getSelectedProvider(); + + if (!selectedProvider) { + return null; // No provider selected } - for (const provider of providers) { - if (await isProviderConfigured(provider)) { - return provider; - } + if (await isProviderConfigured(selectedProvider)) { + return selectedProvider; } - return null; // No providers are properly configured + return null; // Selected provider is not properly configured } /** @@ -163,17 +138,59 @@ export async function getValidModelConfig(provider: ProviderType): Promise<{ mod } /** - * Get the first valid model configuration from the provider precedence list + * Get the model configuration for the currently selected provider */ -export async function getFirstValidModelConfig(): Promise<{ model: string; provider: ProviderType } | null> { - const providers = await getProviderPrecedence(); - - for (const provider of providers) { - const config = await getValidModelConfig(provider); - if (config) { - return config; - } +export async function getSelectedModelConfig(): Promise<{ model: string; provider: ProviderType } | null> { + const selectedProvider = await getSelectedProvider(); + + if (!selectedProvider) { + return null; // No provider selected } - return null; // No valid model configuration found + return await getValidModelConfig(selectedProvider); } + +// Legacy support functions - these maintain backwards compatibility but now use single provider logic +/** + * @deprecated Use getSelectedProvider() instead + */ +export async function getProviderPrecedence(): Promise { + const selected = await getSelectedProvider(); + return selected ? [selected] : []; +} + +/** + * @deprecated Use getSelectedProvider() instead + */ +export async function getPreferredProvider(): Promise { + return await getSelectedProvider(); +} + +/** + * @deprecated Use getSelectedEmbeddingProvider() instead + */ +export async function getEmbeddingProviderPrecedence(): Promise { + const selected = await getSelectedEmbeddingProvider(); + return selected ? [selected] : []; +} + +/** + * @deprecated Use getSelectedEmbeddingProvider() instead + */ +export async function getPreferredEmbeddingProvider(): Promise { + return await getSelectedEmbeddingProvider(); +} + +/** + * @deprecated Use getAvailableSelectedProvider() instead + */ +export async function getFirstAvailableProvider(): Promise { + return await getAvailableSelectedProvider(); +} + +/** + * @deprecated Use getSelectedModelConfig() instead + */ +export async function getFirstValidModelConfig(): Promise<{ model: string; provider: ProviderType } | null> { + return await getSelectedModelConfig(); +} \ No newline at end of file diff --git a/apps/server/src/services/llm/config/configuration_manager.ts b/apps/server/src/services/llm/config/configuration_manager.ts index 5bc9611b8..7eaa435b1 100644 --- a/apps/server/src/services/llm/config/configuration_manager.ts +++ b/apps/server/src/services/llm/config/configuration_manager.ts @@ -50,8 +50,8 @@ export class ConfigurationManager { try { const config: AIConfig = { enabled: await this.getAIEnabled(), - providerPrecedence: await this.getProviderPrecedence(), - embeddingProviderPrecedence: await this.getEmbeddingProviderPrecedence(), + selectedProvider: await this.getSelectedProvider(), + selectedEmbeddingProvider: await this.getSelectedEmbeddingProvider(), defaultModels: await this.getDefaultModels(), providerSettings: await this.getProviderSettings() }; @@ -66,46 +66,28 @@ export class ConfigurationManager { } /** - * Parse provider precedence from string option + * Get the selected AI provider */ - public async getProviderPrecedence(): Promise { + public async getSelectedProvider(): Promise { try { - const precedenceOption = await options.getOption('aiProviderPrecedence'); - const providers = this.parseProviderList(precedenceOption); - - return { - providers: providers as ProviderType[], - defaultProvider: providers.length > 0 ? providers[0] as ProviderType : undefined - }; + const selectedProvider = await options.getOption('aiSelectedProvider'); + return selectedProvider as ProviderType || null; } catch (error) { - log.error(`Error parsing provider precedence: ${error}`); - // Only return known providers if they exist, don't assume defaults - return { - providers: [], - defaultProvider: undefined - }; + log.error(`Error getting selected provider: ${error}`); + return null; } } /** - * Parse embedding provider precedence from string option + * Get the selected embedding provider */ - public async getEmbeddingProviderPrecedence(): Promise { + public async getSelectedEmbeddingProvider(): Promise { try { - const precedenceOption = await options.getOption('embeddingProviderPrecedence'); - const providers = this.parseProviderList(precedenceOption); - - return { - providers: providers as EmbeddingProviderType[], - defaultProvider: providers.length > 0 ? providers[0] as EmbeddingProviderType : undefined - }; + const selectedProvider = await options.getOption('embeddingSelectedProvider'); + return selectedProvider as EmbeddingProviderType || null; } catch (error) { - log.error(`Error parsing embedding provider precedence: ${error}`); - // Don't assume defaults, return empty configuration - return { - providers: [], - defaultProvider: undefined - }; + log.error(`Error getting selected embedding provider: ${error}`); + return null; } } @@ -265,31 +247,29 @@ export class ConfigurationManager { return result; } - // Validate provider precedence - if (config.providerPrecedence.providers.length === 0) { - result.errors.push('No providers configured in precedence list'); + // Validate selected provider + if (!config.selectedProvider) { + result.errors.push('No AI provider selected'); result.isValid = false; - } + } else { + // Validate selected provider settings + const providerConfig = config.providerSettings[config.selectedProvider]; - // Validate provider settings - for (const provider of config.providerPrecedence.providers) { - const providerConfig = config.providerSettings[provider]; - - if (provider === 'openai') { + if (config.selectedProvider === 'openai') { const openaiConfig = providerConfig as OpenAISettings | undefined; if (!openaiConfig?.apiKey) { result.warnings.push('OpenAI API key is not configured'); } } - if (provider === 'anthropic') { + if (config.selectedProvider === 'anthropic') { const anthropicConfig = providerConfig as AnthropicSettings | undefined; if (!anthropicConfig?.apiKey) { result.warnings.push('Anthropic API key is not configured'); } } - if (provider === 'ollama') { + if (config.selectedProvider === 'ollama') { const ollamaConfig = providerConfig as OllamaSettings | undefined; if (!ollamaConfig?.baseUrl) { result.warnings.push('Ollama base URL is not configured'); @@ -297,6 +277,11 @@ export class ConfigurationManager { } } + // Validate selected embedding provider + if (!config.selectedEmbeddingProvider) { + result.warnings.push('No embedding provider selected'); + } + } catch (error) { result.errors.push(`Configuration validation error: ${error}`); result.isValid = false; @@ -356,14 +341,8 @@ export class ConfigurationManager { private getDefaultConfig(): AIConfig { return { enabled: false, - providerPrecedence: { - providers: [], - defaultProvider: undefined - }, - embeddingProviderPrecedence: { - providers: [], - defaultProvider: undefined - }, + selectedProvider: null, + selectedEmbeddingProvider: null, defaultModels: { openai: undefined, anthropic: undefined, diff --git a/apps/server/src/services/llm/context/modules/provider_manager.ts b/apps/server/src/services/llm/context/modules/provider_manager.ts index 8030e3592..6af8ac991 100644 --- a/apps/server/src/services/llm/context/modules/provider_manager.ts +++ b/apps/server/src/services/llm/context/modules/provider_manager.ts @@ -1,51 +1,32 @@ -import options from '../../../options.js'; import log from '../../../log.js'; import { getEmbeddingProvider, getEnabledEmbeddingProviders } from '../../providers/providers.js'; +import { getSelectedEmbeddingProvider } from '../../config/configuration_helpers.js'; /** * Manages embedding providers for context services */ export class ProviderManager { /** - * Get the preferred embedding provider based on user settings - * Tries to use the most appropriate provider in this order: - * 1. User's configured default provider - * 2. OpenAI if API key is set - * 3. Anthropic if API key is set - * 4. Ollama if configured - * 5. Any available provider - * 6. Local provider as fallback + * Get the selected embedding provider based on user settings + * Uses the single provider selection approach * - * @returns The preferred embedding provider or null if none available + * @returns The selected embedding provider or null if none available */ async getPreferredEmbeddingProvider(): Promise { try { - // Try to get providers based on precedence list - const precedenceOption = await options.getOption('embeddingProviderPrecedence'); - let precedenceList: string[] = []; - - if (precedenceOption) { - if (precedenceOption.startsWith('[') && precedenceOption.endsWith(']')) { - precedenceList = JSON.parse(precedenceOption); - } else if (typeof precedenceOption === 'string') { - if (precedenceOption.includes(',')) { - precedenceList = precedenceOption.split(',').map(p => p.trim()); - } else { - precedenceList = [precedenceOption]; - } - } - } - - // Try each provider in the precedence list - for (const providerId of precedenceList) { - const provider = await getEmbeddingProvider(providerId); + // Get the selected embedding provider + const selectedProvider = await getSelectedEmbeddingProvider(); + + if (selectedProvider) { + const provider = await getEmbeddingProvider(selectedProvider); if (provider) { - log.info(`Using embedding provider from precedence list: ${providerId}`); + log.info(`Using selected embedding provider: ${selectedProvider}`); return provider; } + log.info(`Selected embedding provider ${selectedProvider} is not available`); } - // If no provider from precedence list is available, try any enabled provider + // If no provider is selected or available, try any enabled provider const providers = await getEnabledEmbeddingProviders(); if (providers.length > 0) { log.info(`Using available embedding provider: ${providers[0].name}`); diff --git a/apps/server/src/services/llm/index_service.ts b/apps/server/src/services/llm/index_service.ts index f1182495d..bbf372860 100644 --- a/apps/server/src/services/llm/index_service.ts +++ b/apps/server/src/services/llm/index_service.ts @@ -497,40 +497,24 @@ export class IndexService { throw new Error("No embedding providers available"); } - // Get the embedding provider precedence + // Get the selected embedding provider const options = (await import('../options.js')).default; - let preferredProviders: string[] = []; - - const embeddingPrecedence = await options.getOption('embeddingProviderPrecedence'); + const selectedEmbeddingProvider = await options.getOption('embeddingSelectedProvider'); let provider; - if (embeddingPrecedence) { - // Parse the precedence string - if (embeddingPrecedence.startsWith('[') && embeddingPrecedence.endsWith(']')) { - preferredProviders = JSON.parse(embeddingPrecedence); - } else if (typeof embeddingPrecedence === 'string') { - if (embeddingPrecedence.includes(',')) { - preferredProviders = embeddingPrecedence.split(',').map(p => p.trim()); - } else { - preferredProviders = [embeddingPrecedence]; - } - } - - // Find first enabled provider by precedence order - for (const providerName of preferredProviders) { - const matchedProvider = providers.find(p => p.name === providerName); - if (matchedProvider) { - provider = matchedProvider; - break; - } - } - - // If no match found, use first available - if (!provider && providers.length > 0) { + if (selectedEmbeddingProvider) { + // Try to use the selected provider + const enabledProviders = await providerManager.getEnabledEmbeddingProviders(); + provider = enabledProviders.find(p => p.name === selectedEmbeddingProvider); + + if (!provider) { + log.info(`Selected embedding provider ${selectedEmbeddingProvider} is not available, using first enabled provider`); + // Fall back to first enabled provider provider = providers[0]; } } else { - // Default to first available provider + // No provider selected, use first available provider + log.info('No embedding provider selected, using first available provider'); provider = providers[0]; } diff --git a/apps/server/src/services/llm/interfaces/configuration_interfaces.ts b/apps/server/src/services/llm/interfaces/configuration_interfaces.ts index 5a03dc4f1..6adcac977 100644 --- a/apps/server/src/services/llm/interfaces/configuration_interfaces.ts +++ b/apps/server/src/services/llm/interfaces/configuration_interfaces.ts @@ -46,8 +46,8 @@ export interface ModelCapabilities { */ export interface AIConfig { enabled: boolean; - providerPrecedence: ProviderPrecedenceConfig; - embeddingProviderPrecedence: EmbeddingProviderPrecedenceConfig; + selectedProvider: ProviderType | null; + selectedEmbeddingProvider: EmbeddingProviderType | null; defaultModels: Record; providerSettings: ProviderSettings; } @@ -87,7 +87,7 @@ export type ProviderType = 'openai' | 'anthropic' | 'ollama'; /** * Valid embedding provider types */ -export type EmbeddingProviderType = 'openai' | 'ollama' | 'local'; +export type EmbeddingProviderType = 'openai' | 'voyage' | 'ollama' | 'local'; /** * Model identifier with provider prefix (e.g., "openai:gpt-4" or "ollama:llama2") diff --git a/apps/server/src/services/llm/pipeline/stages/model_selection_stage.ts b/apps/server/src/services/llm/pipeline/stages/model_selection_stage.ts index fdecc216e..1be17d7d7 100644 --- a/apps/server/src/services/llm/pipeline/stages/model_selection_stage.ts +++ b/apps/server/src/services/llm/pipeline/stages/model_selection_stage.ts @@ -11,8 +11,7 @@ import type { ServiceProviders } from '../../interfaces/ai_service_interfaces.js // Import new configuration system import { - getProviderPrecedence, - getPreferredProvider, + getSelectedProvider, parseModelIdentifier, getDefaultModelForProvider, createModelConfig @@ -99,22 +98,30 @@ export class ModelSelectionStage extends BasePipelineStage SEARCH_CONSTANTS.CONTEXT.CONTENT_LENGTH.HIGH_THRESHOLD ? 'high' : 'medium'; } - // Set the model and add provider metadata - updatedOptions.model = modelName; - this.addProviderMetadata(updatedOptions, preferredProvider as ServiceProviders, modelName); + // Add provider metadata (model is already set above) + this.addProviderMetadata(updatedOptions, selectedProvider as ServiceProviders, updatedOptions.model); - log.info(`Selected model: ${modelName} from provider: ${preferredProvider} for query complexity: ${queryComplexity}`); + log.info(`Selected model: ${updatedOptions.model} from provider: ${selectedProvider} for query complexity: ${queryComplexity}`); log.info(`[ModelSelectionStage] Final options: ${JSON.stringify({ model: updatedOptions.model, stream: updatedOptions.stream, - provider: preferredProvider, + provider: selectedProvider, enableTools: updatedOptions.enableTools })}`); @@ -210,38 +216,38 @@ export class ModelSelectionStage extends BasePipelineStage { try { - // Use the new configuration system - const providers = await getProviderPrecedence(); + // Use the new single provider configuration system + const selectedProvider = await getSelectedProvider(); - // Use only providers that are available - const availableProviders = providers.filter(provider => - aiServiceManager.isProviderAvailable(provider)); - - if (availableProviders.length === 0) { - throw new Error('No AI providers are available'); + if (!selectedProvider) { + throw new Error('No AI provider is selected. Please select a provider in your AI settings.'); } - // Get the first available provider and its default model - const defaultProvider = availableProviders[0]; - const defaultModel = await getDefaultModelForProvider(defaultProvider); + // Check if the provider is available + if (!aiServiceManager.isProviderAvailable(selectedProvider)) { + throw new Error(`Selected provider ${selectedProvider} is not available`); + } + + // Get the default model for the selected provider + const defaultModel = await getDefaultModelForProvider(selectedProvider); if (!defaultModel) { - throw new Error(`No default model configured for provider ${defaultProvider}. Please configure a default model in your AI settings.`); + throw new Error(`No default model configured for provider ${selectedProvider}. Please configure a default model in your AI settings.`); } // Set provider metadata if (!input.options.providerMetadata) { input.options.providerMetadata = { - provider: defaultProvider as 'openai' | 'anthropic' | 'ollama' | 'local', + provider: selectedProvider as 'openai' | 'anthropic' | 'ollama' | 'local', modelId: defaultModel }; } - log.info(`Selected default model ${defaultModel} from provider ${defaultProvider}`); + log.info(`Selected default model ${defaultModel} from provider ${selectedProvider}`); return defaultModel; } catch (error) { log.error(`Error determining default model: ${error}`); @@ -271,4 +277,126 @@ export class ModelSelectionStage extends BasePipelineStage { + try { + log.info(`Fetching available models for provider ${provider}`); + + // Import server-side options to update the default model + const optionService = (await import('../../../options.js')).default; + + switch (provider) { + case 'openai': + const openaiModels = await this.fetchOpenAIModels(); + if (openaiModels.length > 0) { + // Use the first available model without any preferences + const selectedModel = openaiModels[0]; + + await optionService.setOption('openaiDefaultModel', selectedModel); + log.info(`Set default OpenAI model to: ${selectedModel}`); + return selectedModel; + } + break; + + case 'anthropic': + const anthropicModels = await this.fetchAnthropicModels(); + if (anthropicModels.length > 0) { + // Use the first available model without any preferences + const selectedModel = anthropicModels[0]; + + await optionService.setOption('anthropicDefaultModel', selectedModel); + log.info(`Set default Anthropic model to: ${selectedModel}`); + return selectedModel; + } + break; + + case 'ollama': + const ollamaModels = await this.fetchOllamaModels(); + if (ollamaModels.length > 0) { + // Use the first available model without any preferences + const selectedModel = ollamaModels[0]; + + await optionService.setOption('ollamaDefaultModel', selectedModel); + log.info(`Set default Ollama model to: ${selectedModel}`); + return selectedModel; + } + break; + } + + log.info(`No models available for provider ${provider}`); + return null; + } catch (error) { + log.error(`Error fetching models for provider ${provider}: ${error}`); + return null; + } + } + + /** + * Fetch available OpenAI models + */ + private async fetchOpenAIModels(): Promise { + try { + // Use the provider service to get available models + const aiServiceManager = (await import('../../ai_service_manager.js')).default; + const service = aiServiceManager.getInstance().getService('openai'); + + if (service && typeof (service as any).getAvailableModels === 'function') { + return await (service as any).getAvailableModels(); + } + + // No fallback - return empty array if models can't be fetched + log.info('OpenAI service does not support getAvailableModels method'); + return []; + } catch (error) { + log.error(`Error fetching OpenAI models: ${error}`); + return []; + } + } + + /** + * Fetch available Anthropic models + */ + private async fetchAnthropicModels(): Promise { + try { + // Use the provider service to get available models + const aiServiceManager = (await import('../../ai_service_manager.js')).default; + const service = aiServiceManager.getInstance().getService('anthropic'); + + if (service && typeof (service as any).getAvailableModels === 'function') { + return await (service as any).getAvailableModels(); + } + + // No fallback - return empty array if models can't be fetched + log.info('Anthropic service does not support getAvailableModels method'); + return []; + } catch (error) { + log.error(`Error fetching Anthropic models: ${error}`); + return []; + } + } + + /** + * Fetch available Ollama models + */ + private async fetchOllamaModels(): Promise { + try { + // Use the provider service to get available models + const aiServiceManager = (await import('../../ai_service_manager.js')).default; + const service = aiServiceManager.getInstance().getService('ollama'); + + if (service && typeof (service as any).getAvailableModels === 'function') { + return await (service as any).getAvailableModels(); + } + + // No fallback - return empty array if models can't be fetched + log.info('Ollama service does not support getAvailableModels method'); + return []; + } catch (error) { + log.error(`Error fetching Ollama models: ${error}`); + return []; + } + } } diff --git a/apps/server/src/services/llm/providers/anthropic_service.ts b/apps/server/src/services/llm/providers/anthropic_service.ts index a533acf4a..ed034bdfd 100644 --- a/apps/server/src/services/llm/providers/anthropic_service.ts +++ b/apps/server/src/services/llm/providers/anthropic_service.ts @@ -606,4 +606,12 @@ export class AnthropicService extends BaseAIService { return convertedTools; } + + /** + * Clear cached Anthropic client to force recreation with new settings + */ + clearCache(): void { + this.client = null; + log.info('Anthropic client cache cleared'); + } } diff --git a/apps/server/src/services/llm/providers/ollama_service.ts b/apps/server/src/services/llm/providers/ollama_service.ts index 750118027..4ebbbaa4b 100644 --- a/apps/server/src/services/llm/providers/ollama_service.ts +++ b/apps/server/src/services/llm/providers/ollama_service.ts @@ -526,4 +526,13 @@ export class OllamaService extends BaseAIService { log.info(`Added tool execution feedback: ${toolExecutionStatus.length} statuses`); return updatedMessages; } + + /** + * Clear cached Ollama client to force recreation with new settings + */ + clearCache(): void { + // Ollama service doesn't maintain a persistent client like OpenAI/Anthropic + // but we can clear any future cached state here if needed + log.info('Ollama client cache cleared (no persistent client to clear)'); + } } diff --git a/apps/server/src/services/llm/providers/openai_service.ts b/apps/server/src/services/llm/providers/openai_service.ts index e0633ca1f..09d58498a 100644 --- a/apps/server/src/services/llm/providers/openai_service.ts +++ b/apps/server/src/services/llm/providers/openai_service.ts @@ -257,4 +257,12 @@ export class OpenAIService extends BaseAIService { throw error; } } + + /** + * Clear cached OpenAI client to force recreation with new settings + */ + clearCache(): void { + this.openai = null; + log.info('OpenAI client cache cleared'); + } } diff --git a/apps/server/src/services/options_init.ts b/apps/server/src/services/options_init.ts index 212f47366..817d91fba 100644 --- a/apps/server/src/services/options_init.ts +++ b/apps/server/src/services/options_init.ts @@ -195,26 +195,26 @@ const defaultOptions: DefaultOption[] = [ // AI Options { name: "aiEnabled", value: "false", isSynced: true }, { name: "openaiApiKey", value: "", isSynced: false }, - { name: "openaiDefaultModel", value: "gpt-4o", isSynced: true }, - { name: "openaiEmbeddingModel", value: "text-embedding-3-small", isSynced: true }, + { name: "openaiDefaultModel", value: "", isSynced: true }, + { name: "openaiEmbeddingModel", value: "", isSynced: true }, { name: "openaiBaseUrl", value: "https://api.openai.com/v1", isSynced: true }, { name: "anthropicApiKey", value: "", isSynced: false }, - { name: "anthropicDefaultModel", value: "claude-3-opus-20240229", isSynced: true }, - { name: "voyageEmbeddingModel", value: "voyage-2", isSynced: true }, + { name: "anthropicDefaultModel", value: "", isSynced: true }, + { name: "voyageEmbeddingModel", value: "", isSynced: true }, { name: "voyageApiKey", value: "", isSynced: false }, { name: "anthropicBaseUrl", value: "https://api.anthropic.com/v1", isSynced: true }, { name: "ollamaEnabled", value: "false", isSynced: true }, - { name: "ollamaDefaultModel", value: "llama3", isSynced: true }, + { name: "ollamaDefaultModel", value: "", isSynced: true }, { name: "ollamaBaseUrl", value: "http://localhost:11434", isSynced: true }, - { name: "ollamaEmbeddingModel", value: "nomic-embed-text", isSynced: true }, + { name: "ollamaEmbeddingModel", value: "", isSynced: true }, { name: "embeddingAutoUpdateEnabled", value: "true", isSynced: true }, // Adding missing AI options { name: "aiTemperature", value: "0.7", isSynced: true }, { name: "aiSystemPrompt", value: "", isSynced: true }, - { name: "aiProviderPrecedence", value: "openai,anthropic,ollama", isSynced: true }, + { name: "aiSelectedProvider", value: "openai", isSynced: true }, { name: "embeddingDimensionStrategy", value: "auto", isSynced: true }, - { name: "embeddingProviderPrecedence", value: "openai,voyage,ollama,local", isSynced: true }, + { name: "embeddingSelectedProvider", value: "openai", isSynced: true }, { name: "embeddingSimilarityThreshold", value: "0.75", isSynced: true }, { name: "enableAutomaticIndexing", value: "true", isSynced: true }, { name: "maxNotesPerLlmQuery", value: "3", isSynced: true }, diff --git a/packages/commons/src/lib/options_interface.ts b/packages/commons/src/lib/options_interface.ts index 19125d125..e53d1d9c1 100644 --- a/packages/commons/src/lib/options_interface.ts +++ b/packages/commons/src/lib/options_interface.ts @@ -142,15 +142,14 @@ export interface OptionDefinitions extends KeyboardShortcutsOptions Date: Wed, 4 Jun 2025 20:23:06 +0000 Subject: [PATCH 02/18] fix(llm): have the model_selection_stage use the instance of the aiServiceManager --- .../pipeline/stages/model_selection_stage.ts | 170 +++++------------- 1 file changed, 49 insertions(+), 121 deletions(-) diff --git a/apps/server/src/services/llm/pipeline/stages/model_selection_stage.ts b/apps/server/src/services/llm/pipeline/stages/model_selection_stage.ts index 1be17d7d7..a1c595b18 100644 --- a/apps/server/src/services/llm/pipeline/stages/model_selection_stage.ts +++ b/apps/server/src/services/llm/pipeline/stages/model_selection_stage.ts @@ -100,25 +100,28 @@ export class ModelSelectionStage extends BasePipelineStage { try { - // Use the new single provider configuration system + // Use the same logic as the main process method + const { getValidModelConfig, getSelectedProvider } = await import('../../config/configuration_helpers.js'); const selectedProvider = await getSelectedProvider(); if (!selectedProvider) { throw new Error('No AI provider is selected. Please select a provider in your AI settings.'); } - // Check if the provider is available + // Check if the provider is available through the service manager if (!aiServiceManager.isProviderAvailable(selectedProvider)) { throw new Error(`Selected provider ${selectedProvider} is not available`); } - // Get the default model for the selected provider - const defaultModel = await getDefaultModelForProvider(selectedProvider); - - if (!defaultModel) { + // Try to get a valid model config + const modelConfig = await getValidModelConfig(selectedProvider); + + if (!modelConfig) { throw new Error(`No default model configured for provider ${selectedProvider}. Please configure a default model in your AI settings.`); } @@ -243,12 +248,12 @@ export class ModelSelectionStage extends BasePipelineStage { try { - log.info(`Fetching available models for provider ${provider}`); + log.info(`Getting default model for provider ${provider} using AI service manager`); - // Import server-side options to update the default model - const optionService = (await import('../../../options.js')).default; + // Use the existing AI service manager instead of duplicating API calls + const service = aiServiceManager.getInstance().getService(provider); - switch (provider) { - case 'openai': - const openaiModels = await this.fetchOpenAIModels(); - if (openaiModels.length > 0) { - // Use the first available model without any preferences - const selectedModel = openaiModels[0]; + if (!service || !service.isAvailable()) { + log.info(`Provider ${provider} service is not available`); + return null; + } + + // Check if the service has a method to get available models + if (typeof (service as any).getAvailableModels === 'function') { + try { + const models = await (service as any).getAvailableModels(); + if (models && models.length > 0) { + // Use the first available model - no hardcoded preferences + const selectedModel = models[0]; - await optionService.setOption('openaiDefaultModel', selectedModel); - log.info(`Set default OpenAI model to: ${selectedModel}`); + // Import server-side options to update the default model + const optionService = (await import('../../../options.js')).default; + const optionKey = `${provider}DefaultModel` as const; + + await optionService.setOption(optionKey, selectedModel); + log.info(`Set default ${provider} model to: ${selectedModel}`); return selectedModel; } - break; - - case 'anthropic': - const anthropicModels = await this.fetchAnthropicModels(); - if (anthropicModels.length > 0) { - // Use the first available model without any preferences - const selectedModel = anthropicModels[0]; - - await optionService.setOption('anthropicDefaultModel', selectedModel); - log.info(`Set default Anthropic model to: ${selectedModel}`); - return selectedModel; - } - break; - - case 'ollama': - const ollamaModels = await this.fetchOllamaModels(); - if (ollamaModels.length > 0) { - // Use the first available model without any preferences - const selectedModel = ollamaModels[0]; - - await optionService.setOption('ollamaDefaultModel', selectedModel); - log.info(`Set default Ollama model to: ${selectedModel}`); - return selectedModel; - } - break; + } catch (modelError) { + log.error(`Error fetching models from ${provider} service: ${modelError}`); + } } - log.info(`No models available for provider ${provider}`); + log.info(`Provider ${provider} does not support dynamic model fetching`); return null; } catch (error) { - log.error(`Error fetching models for provider ${provider}: ${error}`); + log.error(`Error getting default model for provider ${provider}: ${error}`); return null; } } - - /** - * Fetch available OpenAI models - */ - private async fetchOpenAIModels(): Promise { - try { - // Use the provider service to get available models - const aiServiceManager = (await import('../../ai_service_manager.js')).default; - const service = aiServiceManager.getInstance().getService('openai'); - - if (service && typeof (service as any).getAvailableModels === 'function') { - return await (service as any).getAvailableModels(); - } - - // No fallback - return empty array if models can't be fetched - log.info('OpenAI service does not support getAvailableModels method'); - return []; - } catch (error) { - log.error(`Error fetching OpenAI models: ${error}`); - return []; - } - } - - /** - * Fetch available Anthropic models - */ - private async fetchAnthropicModels(): Promise { - try { - // Use the provider service to get available models - const aiServiceManager = (await import('../../ai_service_manager.js')).default; - const service = aiServiceManager.getInstance().getService('anthropic'); - - if (service && typeof (service as any).getAvailableModels === 'function') { - return await (service as any).getAvailableModels(); - } - - // No fallback - return empty array if models can't be fetched - log.info('Anthropic service does not support getAvailableModels method'); - return []; - } catch (error) { - log.error(`Error fetching Anthropic models: ${error}`); - return []; - } - } - - /** - * Fetch available Ollama models - */ - private async fetchOllamaModels(): Promise { - try { - // Use the provider service to get available models - const aiServiceManager = (await import('../../ai_service_manager.js')).default; - const service = aiServiceManager.getInstance().getService('ollama'); - - if (service && typeof (service as any).getAvailableModels === 'function') { - return await (service as any).getAvailableModels(); - } - - // No fallback - return empty array if models can't be fetched - log.info('Ollama service does not support getAvailableModels method'); - return []; - } catch (error) { - log.error(`Error fetching Ollama models: ${error}`); - return []; - } - } } From 3dee462476ff02344a0ffc8586402425278310b2 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Wed, 4 Jun 2025 21:32:42 +0000 Subject: [PATCH 03/18] feat(llm): automatically fetch models when provider settings change --- .../options/ai_settings/ai_settings_widget.ts | 134 ++++++++++++++++-- .../options/ai_settings/template.ts | 28 ++-- 2 files changed, 133 insertions(+), 29 deletions(-) diff --git a/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts b/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts index 30f7d3408..ffe4a114c 100644 --- a/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts +++ b/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts @@ -138,19 +138,80 @@ export default class AiSettingsWidget extends OptionsWidget { this.setupChangeHandler('.embedding-update-interval', 'embeddingUpdateInterval'); // Add provider selection change handlers for dynamic settings visibility - this.$widget.find('.ai-selected-provider').on('change', () => { + this.$widget.find('.ai-selected-provider').on('change', async () => { const selectedProvider = this.$widget.find('.ai-selected-provider').val() as string; this.$widget.find('.provider-settings').hide(); if (selectedProvider) { this.$widget.find(`.${selectedProvider}-provider-settings`).show(); + // Automatically fetch models for the newly selected provider + await this.fetchModelsForProvider(selectedProvider, 'chat'); } }); - this.$widget.find('.embedding-selected-provider').on('change', () => { + this.$widget.find('.embedding-selected-provider').on('change', async () => { const selectedProvider = this.$widget.find('.embedding-selected-provider').val() as string; this.$widget.find('.embedding-provider-settings').hide(); if (selectedProvider) { this.$widget.find(`.${selectedProvider}-embedding-provider-settings`).show(); + // Automatically fetch embedding models for the newly selected provider + await this.fetchModelsForProvider(selectedProvider, 'embedding'); + } + }); + + // Add base URL change handlers to trigger model fetching + this.$widget.find('.openai-base-url').on('change', async () => { + const selectedProvider = this.$widget.find('.ai-selected-provider').val() as string; + const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; + if (selectedProvider === 'openai') { + await this.fetchModelsForProvider('openai', 'chat'); + } + if (selectedEmbeddingProvider === 'openai') { + await this.fetchModelsForProvider('openai', 'embedding'); + } + }); + + this.$widget.find('.anthropic-base-url').on('change', async () => { + const selectedProvider = this.$widget.find('.ai-selected-provider').val() as string; + if (selectedProvider === 'anthropic') { + await this.fetchModelsForProvider('anthropic', 'chat'); + } + }); + + this.$widget.find('.ollama-base-url').on('change', async () => { + const selectedProvider = this.$widget.find('.ai-selected-provider').val() as string; + const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; + if (selectedProvider === 'ollama') { + await this.fetchModelsForProvider('ollama', 'chat'); + } + if (selectedEmbeddingProvider === 'ollama') { + await this.fetchModelsForProvider('ollama', 'embedding'); + } + }); + + // Add API key change handlers to trigger model fetching + this.$widget.find('.openai-api-key').on('change', async () => { + const selectedProvider = this.$widget.find('.ai-selected-provider').val() as string; + const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; + if (selectedProvider === 'openai') { + await this.fetchModelsForProvider('openai', 'chat'); + } + if (selectedEmbeddingProvider === 'openai') { + await this.fetchModelsForProvider('openai', 'embedding'); + } + }); + + this.$widget.find('.anthropic-api-key').on('change', async () => { + const selectedProvider = this.$widget.find('.ai-selected-provider').val() as string; + if (selectedProvider === 'anthropic') { + await this.fetchModelsForProvider('anthropic', 'chat'); + } + }); + + this.$widget.find('.voyage-api-key').on('change', async () => { + const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; + if (selectedEmbeddingProvider === 'voyage') { + // Voyage doesn't have dynamic model fetching yet, but we can add it here when implemented + console.log('Voyage API key changed - model fetching not yet implemented'); } }); @@ -446,6 +507,49 @@ export default class AiSettingsWidget extends OptionsWidget { } } + /** + * Set model dropdown value, adding the option if it doesn't exist + */ + setModelDropdownValue(selector: string, value: string | undefined) { + if (!this.$widget || !value) return; + + const $dropdown = this.$widget.find(selector); + + // Check if the value already exists as an option + if ($dropdown.find(`option[value="${value}"]`).length === 0) { + // Add the custom value as an option + $dropdown.append(``); + } + + // Set the value + $dropdown.val(value); + } + + /** + * Fetch models for a specific provider and model type + */ + async fetchModelsForProvider(provider: string, modelType: 'chat' | 'embedding') { + if (!this.providerService) return; + + try { + switch (provider) { + case 'openai': + this.openaiModelsRefreshed = await this.providerService.refreshOpenAIModels(false, this.openaiModelsRefreshed); + break; + case 'anthropic': + this.anthropicModelsRefreshed = await this.providerService.refreshAnthropicModels(false, this.anthropicModelsRefreshed); + break; + case 'ollama': + this.ollamaModelsRefreshed = await this.providerService.refreshOllamaModels(false, this.ollamaModelsRefreshed); + break; + default: + console.log(`Model fetching not implemented for provider: ${provider}`); + } + } catch (error) { + console.error(`Error fetching models for ${provider}:`, error); + } + } + /** * Update provider settings visibility based on selected providers */ @@ -470,7 +574,7 @@ export default class AiSettingsWidget extends OptionsWidget { /** * Called when the options have been loaded from the server */ - optionsLoaded(options: OptionMap) { + async optionsLoaded(options: OptionMap) { if (!this.$widget) return; // AI Options @@ -482,22 +586,22 @@ export default class AiSettingsWidget extends OptionsWidget { // OpenAI Section this.$widget.find('.openai-api-key').val(options.openaiApiKey || ''); this.$widget.find('.openai-base-url').val(options.openaiBaseUrl || 'https://api.openai.com/v1'); - this.$widget.find('.openai-default-model').val(options.openaiDefaultModel || ''); - this.$widget.find('.openai-embedding-model').val(options.openaiEmbeddingModel || ''); + this.setModelDropdownValue('.openai-default-model', options.openaiDefaultModel); + this.setModelDropdownValue('.openai-embedding-model', options.openaiEmbeddingModel); // Anthropic Section this.$widget.find('.anthropic-api-key').val(options.anthropicApiKey || ''); this.$widget.find('.anthropic-base-url').val(options.anthropicBaseUrl || 'https://api.anthropic.com'); - this.$widget.find('.anthropic-default-model').val(options.anthropicDefaultModel || ''); + this.setModelDropdownValue('.anthropic-default-model', options.anthropicDefaultModel); // Voyage Section this.$widget.find('.voyage-api-key').val(options.voyageApiKey || ''); - this.$widget.find('.voyage-embedding-model').val(options.voyageEmbeddingModel || ''); + this.setModelDropdownValue('.voyage-embedding-model', options.voyageEmbeddingModel); // Ollama Section this.$widget.find('.ollama-base-url').val(options.ollamaBaseUrl || 'http://localhost:11434'); - this.$widget.find('.ollama-default-model').val(options.ollamaDefaultModel || ''); - this.$widget.find('.ollama-embedding-model').val(options.ollamaEmbeddingModel || ''); + this.setModelDropdownValue('.ollama-default-model', options.ollamaDefaultModel); + this.setModelDropdownValue('.ollama-embedding-model', options.ollamaEmbeddingModel); // Embedding Options this.$widget.find('.embedding-selected-provider').val(options.embeddingSelectedProvider || 'openai'); @@ -512,6 +616,18 @@ export default class AiSettingsWidget extends OptionsWidget { // Show/hide provider settings based on selected providers this.updateProviderSettingsVisibility(); + // Automatically fetch models for currently selected providers + const selectedAiProvider = this.$widget.find('.ai-selected-provider').val() as string; + const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; + + if (selectedAiProvider) { + await this.fetchModelsForProvider(selectedAiProvider, 'chat'); + } + + if (selectedEmbeddingProvider) { + await this.fetchModelsForProvider(selectedEmbeddingProvider, 'embedding'); + } + // Display validation warnings this.displayValidationWarnings(); } diff --git a/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts b/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts index af00d4474..5115c2144 100644 --- a/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts +++ b/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts @@ -93,9 +93,7 @@ export const TPL = `
${t("ai_llm.openai_model_description")}
@@ -104,8 +102,7 @@ export const TPL = `
${t("ai_llm.openai_embedding_model_description")}
@@ -135,9 +132,7 @@ export const TPL = `
${t("ai_llm.anthropic_model_description")}
@@ -162,9 +157,7 @@ export const TPL = `
${t("ai_llm.ollama_model_description")}
@@ -173,8 +166,7 @@ export const TPL = `
${t("ai_llm.ollama_embedding_model_description")}
@@ -221,8 +213,7 @@ export const TPL = `
${t("ai_llm.openai_embedding_model_description")}
@@ -247,9 +238,7 @@ export const TPL = `
${t("ai_llm.voyage_embedding_model_description")}
@@ -267,8 +256,7 @@ export const TPL = `
${t("ai_llm.ollama_embedding_model_description")}
From 63722a28a28b7e17bc5cb7437306a2804bad5d71 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Wed, 4 Jun 2025 22:30:16 +0000 Subject: [PATCH 04/18] feat(llm): also add embeddings options for embedding creation --- .../options/ai_settings/ai_settings_widget.ts | 51 +++++++++++++++++-- .../options/ai_settings/template.ts | 26 +++++++++- apps/server/src/services/options_init.ts | 6 +++ packages/commons/src/lib/options_interface.ts | 4 ++ 4 files changed, 81 insertions(+), 6 deletions(-) diff --git a/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts b/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts index ffe4a114c..5b2480c1a 100644 --- a/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts +++ b/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts @@ -83,11 +83,17 @@ export default class AiSettingsWidget extends OptionsWidget { // Voyage options this.setupChangeHandler('.voyage-api-key', 'voyageApiKey'); this.setupChangeHandler('.voyage-embedding-model', 'voyageEmbeddingModel'); + this.setupChangeHandler('.voyage-embedding-base-url', 'voyageEmbeddingBaseUrl'); // Ollama options this.setupChangeHandler('.ollama-base-url', 'ollamaBaseUrl'); this.setupChangeHandler('.ollama-default-model', 'ollamaDefaultModel'); this.setupChangeHandler('.ollama-embedding-model', 'ollamaEmbeddingModel'); + this.setupChangeHandler('.ollama-embedding-base-url', 'ollamaEmbeddingBaseUrl'); + + // Embedding-specific provider options + this.setupChangeHandler('.openai-embedding-api-key', 'openaiEmbeddingApiKey', true); + this.setupChangeHandler('.openai-embedding-base-url', 'openaiEmbeddingBaseUrl', true); const $refreshModels = this.$widget.find('.refresh-models'); $refreshModels.on('click', async () => { @@ -215,6 +221,37 @@ export default class AiSettingsWidget extends OptionsWidget { } }); + // Add embedding base URL change handlers to trigger model fetching + this.$widget.find('.openai-embedding-base-url').on('change', async () => { + const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; + if (selectedEmbeddingProvider === 'openai') { + await this.fetchModelsForProvider('openai', 'embedding'); + } + }); + + this.$widget.find('.voyage-embedding-base-url').on('change', async () => { + const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; + if (selectedEmbeddingProvider === 'voyage') { + // Voyage doesn't have dynamic model fetching yet, but we can add it here when implemented + console.log('Voyage embedding base URL changed - model fetching not yet implemented'); + } + }); + + this.$widget.find('.ollama-embedding-base-url').on('change', async () => { + const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; + if (selectedEmbeddingProvider === 'ollama') { + await this.fetchModelsForProvider('ollama', 'embedding'); + } + }); + + // Add embedding API key change handlers to trigger model fetching + this.$widget.find('.openai-embedding-api-key').on('change', async () => { + const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; + if (selectedEmbeddingProvider === 'openai') { + await this.fetchModelsForProvider('openai', 'embedding'); + } + }); + // No sortable behavior needed anymore // Embedding stats refresh button @@ -514,13 +551,13 @@ export default class AiSettingsWidget extends OptionsWidget { if (!this.$widget || !value) return; const $dropdown = this.$widget.find(selector); - + // Check if the value already exists as an option if ($dropdown.find(`option[value="${value}"]`).length === 0) { // Add the custom value as an option $dropdown.append(``); } - + // Set the value $dropdown.val(value); } @@ -596,13 +633,19 @@ export default class AiSettingsWidget extends OptionsWidget { // Voyage Section this.$widget.find('.voyage-api-key').val(options.voyageApiKey || ''); + this.$widget.find('.voyage-embedding-base-url').val(options.voyageEmbeddingBaseUrl || 'https://api.voyageai.com/v1'); this.setModelDropdownValue('.voyage-embedding-model', options.voyageEmbeddingModel); // Ollama Section this.$widget.find('.ollama-base-url').val(options.ollamaBaseUrl || 'http://localhost:11434'); + this.$widget.find('.ollama-embedding-base-url').val(options.ollamaEmbeddingBaseUrl || 'http://localhost:11434'); this.setModelDropdownValue('.ollama-default-model', options.ollamaDefaultModel); this.setModelDropdownValue('.ollama-embedding-model', options.ollamaEmbeddingModel); + // Embedding-specific provider options + this.$widget.find('.openai-embedding-api-key').val(options.openaiEmbeddingApiKey || ''); + this.$widget.find('.openai-embedding-base-url').val(options.openaiEmbeddingBaseUrl || 'https://api.openai.com/v1'); + // Embedding Options this.$widget.find('.embedding-selected-provider').val(options.embeddingSelectedProvider || 'openai'); this.$widget.find('.embedding-auto-update-enabled').prop('checked', options.embeddingAutoUpdateEnabled !== 'false'); @@ -619,11 +662,11 @@ export default class AiSettingsWidget extends OptionsWidget { // Automatically fetch models for currently selected providers const selectedAiProvider = this.$widget.find('.ai-selected-provider').val() as string; const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; - + if (selectedAiProvider) { await this.fetchModelsForProvider(selectedAiProvider, 'chat'); } - + if (selectedEmbeddingProvider) { await this.fetchModelsForProvider(selectedEmbeddingProvider, 'embedding'); } diff --git a/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts b/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts index 5115c2144..4778930af 100644 --- a/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts +++ b/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts @@ -210,6 +210,18 @@ export const TPL = `
${t("ai_llm.openai_embedding_settings")}
+
+ + +
${t("ai_llm.openai_embedding_api_key_description")}
+
+ +
+ + +
${t("ai_llm.openai_embedding_url_description")}
+
+
${t("ai_llm.openai_embedding_model_description")}
-
${t("ai_llm.openai_embedding_shared_settings")}
@@ -235,6 +246,12 @@ export const TPL = `
${t("ai_llm.voyage_api_key_description")}
+
+ + +
${t("ai_llm.voyage_embedding_url_description")}
+
+
+
${t("ai_llm.ollama_embedding_url_description")}
+
+
${t("ai_llm.ollama_embedding_model_description")}
-
${t("ai_llm.ollama_embedding_shared_settings")}
diff --git a/apps/server/src/services/options_init.ts b/apps/server/src/services/options_init.ts index 817d91fba..5311a67f0 100644 --- a/apps/server/src/services/options_init.ts +++ b/apps/server/src/services/options_init.ts @@ -209,6 +209,12 @@ const defaultOptions: DefaultOption[] = [ { name: "ollamaEmbeddingModel", value: "", isSynced: true }, { name: "embeddingAutoUpdateEnabled", value: "true", isSynced: true }, + // Embedding-specific provider options + { name: "openaiEmbeddingApiKey", value: "", isSynced: false }, + { name: "openaiEmbeddingBaseUrl", value: "https://api.openai.com/v1", isSynced: true }, + { name: "voyageEmbeddingBaseUrl", value: "https://api.voyageai.com/v1", isSynced: true }, + { name: "ollamaEmbeddingBaseUrl", value: "http://localhost:11434", isSynced: true }, + // Adding missing AI options { name: "aiTemperature", value: "0.7", isSynced: true }, { name: "aiSystemPrompt", value: "", isSynced: true }, diff --git a/packages/commons/src/lib/options_interface.ts b/packages/commons/src/lib/options_interface.ts index e53d1d9c1..45faf4d79 100644 --- a/packages/commons/src/lib/options_interface.ts +++ b/packages/commons/src/lib/options_interface.ts @@ -131,16 +131,20 @@ export interface OptionDefinitions extends KeyboardShortcutsOptions Date: Wed, 4 Jun 2025 22:58:20 +0000 Subject: [PATCH 05/18] feat(llm): also have the embedding provider settings be changeable --- apps/server/src/routes/api/options.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/server/src/routes/api/options.ts b/apps/server/src/routes/api/options.ts index ce6275c8e..edfa75d34 100644 --- a/apps/server/src/routes/api/options.ts +++ b/apps/server/src/routes/api/options.ts @@ -100,14 +100,18 @@ const ALLOWED_OPTIONS = new Set([ "openaiBaseUrl", "openaiDefaultModel", "openaiEmbeddingModel", + "openaiEmbeddingApiKey", + "openaiEmbeddingBaseUrl", "anthropicApiKey", "anthropicBaseUrl", "anthropicDefaultModel", "voyageApiKey", "voyageEmbeddingModel", + "voyageEmbeddingBaseUrl", "ollamaBaseUrl", "ollamaDefaultModel", "ollamaEmbeddingModel", + "ollamaEmbeddingBaseUrl", "embeddingAutoUpdateEnabled", "embeddingDimensionStrategy", "embeddingSelectedProvider", From 5db514e245b1e130afb2d4484cb09be8e02f5582 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Wed, 4 Jun 2025 23:02:10 +0000 Subject: [PATCH 06/18] fix(llm): fix the buggy embedding selection dropdown --- .../type_widgets/options/ai_settings/ai_settings_widget.ts | 2 ++ .../src/widgets/type_widgets/options/ai_settings/template.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts b/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts index 5b2480c1a..67cc80cbb 100644 --- a/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts +++ b/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts @@ -89,6 +89,7 @@ export default class AiSettingsWidget extends OptionsWidget { this.setupChangeHandler('.ollama-base-url', 'ollamaBaseUrl'); this.setupChangeHandler('.ollama-default-model', 'ollamaDefaultModel'); this.setupChangeHandler('.ollama-embedding-model', 'ollamaEmbeddingModel'); + this.setupChangeHandler('.ollama-chat-embedding-model', 'ollamaEmbeddingModel'); this.setupChangeHandler('.ollama-embedding-base-url', 'ollamaEmbeddingBaseUrl'); // Embedding-specific provider options @@ -641,6 +642,7 @@ export default class AiSettingsWidget extends OptionsWidget { this.$widget.find('.ollama-embedding-base-url').val(options.ollamaEmbeddingBaseUrl || 'http://localhost:11434'); this.setModelDropdownValue('.ollama-default-model', options.ollamaDefaultModel); this.setModelDropdownValue('.ollama-embedding-model', options.ollamaEmbeddingModel); + this.setModelDropdownValue('.ollama-chat-embedding-model', options.ollamaEmbeddingModel); // Embedding-specific provider options this.$widget.find('.openai-embedding-api-key').val(options.openaiEmbeddingApiKey || ''); diff --git a/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts b/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts index 4778930af..ac09f6b97 100644 --- a/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts +++ b/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts @@ -165,7 +165,7 @@ export const TPL = `
-
${t("ai_llm.ollama_embedding_model_description")}
From 49e123f39990f6fd33a2a7b7b2ef9ff9e2ad4b3d Mon Sep 17 00:00:00 2001 From: perf3ct Date: Thu, 5 Jun 2025 18:47:25 +0000 Subject: [PATCH 07/18] feat(llm): create endpoints for starting/stopping embeddings --- .../options/ai_settings/ai_settings_widget.ts | 29 +++++++ apps/server/src/routes/api/embeddings.ts | 47 +++++++++++- apps/server/src/routes/routes.ts | 2 + .../src/services/llm/ai_service_manager.ts | 24 +++++- .../src/services/llm/embeddings/events.ts | 22 +++++- .../src/services/llm/embeddings/index.ts | 2 + apps/server/src/services/llm/index_service.ts | 75 +++++++++++++++++++ 7 files changed, 197 insertions(+), 4 deletions(-) diff --git a/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts b/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts index 67cc80cbb..4f63bfcb1 100644 --- a/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts +++ b/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts @@ -51,6 +51,35 @@ export default class AiSettingsWidget extends OptionsWidget { await this.updateOption(optionName, value); + // Special handling for aiEnabled option + if (optionName === 'aiEnabled') { + try { + const isEnabled = value === 'true'; + + if (isEnabled) { + // Start embedding generation + await server.post('llm/embeddings/start'); + toastService.showMessage(t("ai_llm.embeddings_started") || "Embedding generation started"); + + // Start polling for stats updates + this.refreshEmbeddingStats(); + } else { + // Stop embedding generation + await server.post('llm/embeddings/stop'); + toastService.showMessage(t("ai_llm.embeddings_stopped") || "Embedding generation stopped"); + + // Clear any active polling intervals + if (this.indexRebuildRefreshInterval) { + clearInterval(this.indexRebuildRefreshInterval); + this.indexRebuildRefreshInterval = null; + } + } + } catch (error) { + console.error('Error toggling embeddings:', error); + toastService.showError(t("ai_llm.embeddings_toggle_error") || "Error toggling embeddings"); + } + } + if (validateAfter) { await this.displayValidationWarnings(); } diff --git a/apps/server/src/routes/api/embeddings.ts b/apps/server/src/routes/api/embeddings.ts index 012a9c82f..8fd9b475a 100644 --- a/apps/server/src/routes/api/embeddings.ts +++ b/apps/server/src/routes/api/embeddings.ts @@ -782,6 +782,49 @@ async function getIndexRebuildStatus(req: Request, res: Response) { }; } +/** + * Start embedding generation when AI is enabled + */ +async function startEmbeddings(req: Request, res: Response) { + try { + log.info("Starting embedding generation system"); + + // Initialize the index service if not already initialized + await indexService.initialize(); + + // Start automatic indexing + await indexService.startEmbeddingGeneration(); + + return { + success: true, + message: "Embedding generation started" + }; + } catch (error: any) { + log.error(`Error starting embeddings: ${error.message || 'Unknown error'}`); + throw new Error(`Failed to start embeddings: ${error.message || 'Unknown error'}`); + } +} + +/** + * Stop embedding generation when AI is disabled + */ +async function stopEmbeddings(req: Request, res: Response) { + try { + log.info("Stopping embedding generation system"); + + // Stop automatic indexing + await indexService.stopEmbeddingGeneration(); + + return { + success: true, + message: "Embedding generation stopped" + }; + } catch (error: any) { + log.error(`Error stopping embeddings: ${error.message || 'Unknown error'}`); + throw new Error(`Failed to stop embeddings: ${error.message || 'Unknown error'}`); + } +} + export default { findSimilarNotes, searchByText, @@ -794,5 +837,7 @@ export default { retryFailedNote, retryAllFailedNotes, rebuildIndex, - getIndexRebuildStatus + getIndexRebuildStatus, + startEmbeddings, + stopEmbeddings }; diff --git a/apps/server/src/routes/routes.ts b/apps/server/src/routes/routes.ts index d9ac2c2f8..73beb28e2 100644 --- a/apps/server/src/routes/routes.ts +++ b/apps/server/src/routes/routes.ts @@ -400,6 +400,8 @@ function register(app: express.Application) { asyncApiRoute(PST, "/api/llm/embeddings/retry-all-failed", embeddingsRoute.retryAllFailedNotes); asyncApiRoute(PST, "/api/llm/embeddings/rebuild-index", embeddingsRoute.rebuildIndex); asyncApiRoute(GET, "/api/llm/embeddings/index-rebuild-status", embeddingsRoute.getIndexRebuildStatus); + asyncApiRoute(PST, "/api/llm/embeddings/start", embeddingsRoute.startEmbeddings); + asyncApiRoute(PST, "/api/llm/embeddings/stop", embeddingsRoute.stopEmbeddings); // LLM provider endpoints - moved under /api/llm/providers hierarchy asyncApiRoute(GET, "/api/llm/providers/ollama/models", ollamaRoute.listModels); diff --git a/apps/server/src/services/llm/ai_service_manager.ts b/apps/server/src/services/llm/ai_service_manager.ts index aeb6e9763..9e84d1e28 100644 --- a/apps/server/src/services/llm/ai_service_manager.ts +++ b/apps/server/src/services/llm/ai_service_manager.ts @@ -605,6 +605,7 @@ export class AIServiceManager implements IAIServiceManager { private setupProviderChangeListener(): void { // List of AI-related options that should trigger service recreation const aiRelatedOptions = [ + 'aiEnabled', 'aiSelectedProvider', 'embeddingSelectedProvider', 'openaiApiKey', @@ -618,10 +619,29 @@ export class AIServiceManager implements IAIServiceManager { 'voyageApiKey' ]; - eventService.subscribe(['entityChanged'], ({ entityName, entity }) => { + eventService.subscribe(['entityChanged'], async ({ entityName, entity }) => { if (entityName === 'options' && entity && aiRelatedOptions.includes(entity.name)) { log.info(`AI-related option '${entity.name}' changed, recreating LLM services`); - this.recreateServices(); + + // Special handling for aiEnabled toggle + if (entity.name === 'aiEnabled') { + const isEnabled = entity.value === 'true'; + + if (isEnabled) { + log.info('AI features enabled, initializing AI service and embeddings'); + // Initialize the AI service + await this.initialize(); + // Initialize embeddings through index service + await indexService.startEmbeddingGeneration(); + } else { + log.info('AI features disabled, stopping embeddings'); + // Stop embeddings through index service + await indexService.stopEmbeddingGeneration(); + } + } else { + // For other AI-related options, just recreate services + this.recreateServices(); + } } }); } diff --git a/apps/server/src/services/llm/embeddings/events.ts b/apps/server/src/services/llm/embeddings/events.ts index a078b2c32..2b8eac7d9 100644 --- a/apps/server/src/services/llm/embeddings/events.ts +++ b/apps/server/src/services/llm/embeddings/events.ts @@ -9,6 +9,9 @@ import becca from "../../../becca/becca.js"; // Add mutex to prevent concurrent processing let isProcessingEmbeddings = false; +// Store interval reference for cleanup +let backgroundProcessingInterval: NodeJS.Timeout | null = null; + /** * Setup event listeners for embedding-related events */ @@ -53,9 +56,15 @@ export function setupEmbeddingEventListeners() { * Setup background processing of the embedding queue */ export async function setupEmbeddingBackgroundProcessing() { + // Clear any existing interval + if (backgroundProcessingInterval) { + clearInterval(backgroundProcessingInterval); + backgroundProcessingInterval = null; + } + const interval = parseInt(await options.getOption('embeddingUpdateInterval') || '200', 10); - setInterval(async () => { + backgroundProcessingInterval = setInterval(async () => { try { // Skip if already processing if (isProcessingEmbeddings) { @@ -78,6 +87,17 @@ export async function setupEmbeddingBackgroundProcessing() { }, interval); } +/** + * Stop background processing of the embedding queue + */ +export function stopEmbeddingBackgroundProcessing() { + if (backgroundProcessingInterval) { + clearInterval(backgroundProcessingInterval); + backgroundProcessingInterval = null; + log.info("Embedding background processing stopped"); + } +} + /** * Initialize embeddings system */ diff --git a/apps/server/src/services/llm/embeddings/index.ts b/apps/server/src/services/llm/embeddings/index.ts index 89d0f711e..2757f8808 100644 --- a/apps/server/src/services/llm/embeddings/index.ts +++ b/apps/server/src/services/llm/embeddings/index.ts @@ -58,6 +58,7 @@ export const processNoteWithChunking = async ( export const { setupEmbeddingEventListeners, setupEmbeddingBackgroundProcessing, + stopEmbeddingBackgroundProcessing, initEmbeddings } = events; @@ -100,6 +101,7 @@ export default { // Event handling setupEmbeddingEventListeners: events.setupEmbeddingEventListeners, setupEmbeddingBackgroundProcessing: events.setupEmbeddingBackgroundProcessing, + stopEmbeddingBackgroundProcessing: events.stopEmbeddingBackgroundProcessing, initEmbeddings: events.initEmbeddings, // Stats and maintenance diff --git a/apps/server/src/services/llm/index_service.ts b/apps/server/src/services/llm/index_service.ts index bbf372860..992739947 100644 --- a/apps/server/src/services/llm/index_service.ts +++ b/apps/server/src/services/llm/index_service.ts @@ -837,6 +837,81 @@ export class IndexService { return false; } } + + /** + * Start embedding generation (called when AI is enabled) + */ + async startEmbeddingGeneration() { + try { + log.info("Starting embedding generation system"); + + // Re-initialize if needed + if (!this.initialized) { + await this.initialize(); + } + + const aiEnabled = options.getOptionOrNull('aiEnabled') === "true"; + if (!aiEnabled) { + log.error("Cannot start embedding generation - AI features are disabled"); + throw new Error("AI features must be enabled first"); + } + + // Check if this instance should process embeddings + const embeddingLocation = await options.getOption('embeddingGenerationLocation') || 'client'; + const isSyncServer = await this.isSyncServerForEmbeddings(); + const shouldProcessEmbeddings = embeddingLocation === 'client' || isSyncServer; + + if (!shouldProcessEmbeddings) { + log.info("This instance is not configured to process embeddings"); + return; + } + + // Setup automatic indexing if enabled + if (await options.getOptionBool('embeddingAutoUpdateEnabled')) { + this.setupAutomaticIndexing(); + log.info(`Automatic embedding indexing started ${isSyncServer ? 'as sync server' : 'as client'}`); + } + + // Re-initialize event listeners + this.setupEventListeners(); + + // Start processing the queue immediately + await this.runBatchIndexing(20); + + log.info("Embedding generation started successfully"); + } catch (error: any) { + log.error(`Error starting embedding generation: ${error.message || "Unknown error"}`); + throw error; + } + } + + /** + * Stop embedding generation (called when AI is disabled) + */ + async stopEmbeddingGeneration() { + try { + log.info("Stopping embedding generation system"); + + // Clear automatic indexing interval + if (this.automaticIndexingInterval) { + clearInterval(this.automaticIndexingInterval); + this.automaticIndexingInterval = undefined; + log.info("Automatic indexing stopped"); + } + + // Stop the background processing from embeddings/events.ts + vectorStore.stopEmbeddingBackgroundProcessing(); + + // Mark as not indexing + this.indexingInProgress = false; + this.indexRebuildInProgress = false; + + log.info("Embedding generation stopped successfully"); + } catch (error: any) { + log.error(`Error stopping embedding generation: ${error.message || "Unknown error"}`); + throw error; + } + } } // Create singleton instance From c1b10d70b8b65a6d9fa8853972bf66acf09a0d1f Mon Sep 17 00:00:00 2001 From: perf3ct Date: Thu, 5 Jun 2025 18:59:32 +0000 Subject: [PATCH 08/18] feat(llm): also add functions to clear/unregister embedding providers --- .../src/services/llm/embeddings/init.ts | 5 ++- apps/server/src/services/llm/index_service.ts | 17 +++++++--- .../src/services/llm/providers/providers.ts | 33 ++++++++++++++++--- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/apps/server/src/services/llm/embeddings/init.ts b/apps/server/src/services/llm/embeddings/init.ts index 50724c05e..94188fa69 100644 --- a/apps/server/src/services/llm/embeddings/init.ts +++ b/apps/server/src/services/llm/embeddings/init.ts @@ -43,11 +43,10 @@ export async function initializeEmbeddings() { // Reset any stuck embedding queue items from previous server shutdown await resetStuckEmbeddingQueue(); - // Initialize default embedding providers - await providerManager.initializeDefaultProviders(); - // Start the embedding system if AI is enabled if (await options.getOptionBool('aiEnabled')) { + // Initialize default embedding providers when AI is enabled + await providerManager.initializeDefaultProviders(); await initEmbeddings(); log.info("Embedding system initialized successfully."); } else { diff --git a/apps/server/src/services/llm/index_service.ts b/apps/server/src/services/llm/index_service.ts index 992739947..a179431ee 100644 --- a/apps/server/src/services/llm/index_service.ts +++ b/apps/server/src/services/llm/index_service.ts @@ -845,17 +845,21 @@ export class IndexService { try { log.info("Starting embedding generation system"); - // Re-initialize if needed - if (!this.initialized) { - await this.initialize(); - } - const aiEnabled = options.getOptionOrNull('aiEnabled') === "true"; if (!aiEnabled) { log.error("Cannot start embedding generation - AI features are disabled"); throw new Error("AI features must be enabled first"); } + // Re-initialize providers first in case they weren't available when server started + log.info("Re-initializing embedding providers"); + await providerManager.initializeDefaultProviders(); + + // Re-initialize if needed + if (!this.initialized) { + await this.initialize(); + } + // Check if this instance should process embeddings const embeddingLocation = await options.getOption('embeddingGenerationLocation') || 'client'; const isSyncServer = await this.isSyncServerForEmbeddings(); @@ -902,6 +906,9 @@ export class IndexService { // Stop the background processing from embeddings/events.ts vectorStore.stopEmbeddingBackgroundProcessing(); + // Clear all embedding providers to clean up resources + providerManager.clearAllEmbeddingProviders(); + // Mark as not indexing this.indexingInProgress = false; this.indexRebuildInProgress = false; diff --git a/apps/server/src/services/llm/providers/providers.ts b/apps/server/src/services/llm/providers/providers.ts index f4d69801c..5b4955ce6 100644 --- a/apps/server/src/services/llm/providers/providers.ts +++ b/apps/server/src/services/llm/providers/providers.ts @@ -86,6 +86,29 @@ export function registerEmbeddingProvider(provider: EmbeddingProvider) { log.info(`Registered embedding provider: ${provider.name}`); } +/** + * Unregister an embedding provider + */ +export function unregisterEmbeddingProvider(name: string): boolean { + const existed = providers.has(name); + if (existed) { + providers.delete(name); + log.info(`Unregistered embedding provider: ${name}`); + } + return existed; +} + +/** + * Clear all embedding providers + */ +export function clearAllEmbeddingProviders(): void { + const providerNames = Array.from(providers.keys()); + providers.clear(); + if (providerNames.length > 0) { + log.info(`Cleared all embedding providers: ${providerNames.join(', ')}`); + } +} + /** * Get all registered embedding providers */ @@ -296,9 +319,9 @@ export async function initializeDefaultProviders() { } } - // Register Ollama provider if base URL is configured - const ollamaBaseUrl = await options.getOption('ollamaBaseUrl'); - if (ollamaBaseUrl) { + // Register Ollama embedding provider if embedding base URL is configured + const ollamaEmbeddingBaseUrl = await options.getOption('ollamaEmbeddingBaseUrl'); + if (ollamaEmbeddingBaseUrl) { // Use specific embedding models if available const embeddingModel = await options.getOption('ollamaEmbeddingModel'); @@ -308,7 +331,7 @@ export async function initializeDefaultProviders() { model: embeddingModel, dimension: 768, // Initial value, will be updated during initialization type: 'float32', - baseUrl: ollamaBaseUrl + baseUrl: ollamaEmbeddingBaseUrl }); // Register the provider @@ -362,6 +385,8 @@ export async function initializeDefaultProviders() { export default { registerEmbeddingProvider, + unregisterEmbeddingProvider, + clearAllEmbeddingProviders, getEmbeddingProviders, getEmbeddingProvider, getEnabledEmbeddingProviders, From bb8a374ab8a2655f5f8774aa40f08eb8630f81a2 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Thu, 5 Jun 2025 19:27:45 +0000 Subject: [PATCH 09/18] feat(llm): transition from initializing LLM providers, to creating them on demand --- .../src/services/llm/ai_service_manager.ts | 197 ++++++++------ apps/server/src/services/llm/context/index.ts | 4 +- .../src/services/llm/embeddings/init.ts | 3 +- apps/server/src/services/llm/index_service.ts | 11 +- .../llm/interfaces/ai_service_interfaces.ts | 2 +- .../stages/context_extraction_stage.ts | 2 +- .../pipeline/stages/llm_completion_stage.ts | 2 +- .../pipeline/stages/model_selection_stage.ts | 2 +- .../src/services/llm/providers/providers.ts | 247 ++++++++---------- .../llm/tools/note_summarization_tool.ts | 7 +- .../services/llm/tools/relationship_tool.ts | 11 +- .../services/llm/tools/search_notes_tool.ts | 19 +- 12 files changed, 246 insertions(+), 261 deletions(-) diff --git a/apps/server/src/services/llm/ai_service_manager.ts b/apps/server/src/services/llm/ai_service_manager.ts index 9e84d1e28..f054dff57 100644 --- a/apps/server/src/services/llm/ai_service_manager.ts +++ b/apps/server/src/services/llm/ai_service_manager.ts @@ -43,11 +43,7 @@ interface NoteContext { } export class AIServiceManager implements IAIServiceManager { - private services: Record = { - openai: new OpenAIService(), - anthropic: new AnthropicService(), - ollama: new OllamaService() - }; + private services: Partial> = {}; private providerOrder: ServiceProviders[] = []; // Will be populated from configuration private initialized = false; @@ -183,9 +179,42 @@ export class AIServiceManager implements IAIServiceManager { */ getAvailableProviders(): ServiceProviders[] { this.ensureInitialized(); - return Object.entries(this.services) - .filter(([_, service]) => service.isAvailable()) - .map(([key, _]) => key as ServiceProviders); + + const allProviders: ServiceProviders[] = ['openai', 'anthropic', 'ollama']; + const availableProviders: ServiceProviders[] = []; + + for (const providerName of allProviders) { + // Use a sync approach - check if we can create the provider + const service = this.services[providerName]; + if (service && service.isAvailable()) { + availableProviders.push(providerName); + } else { + // For providers not yet created, check configuration to see if they would be available + try { + switch (providerName) { + case 'openai': + if (options.getOption('openaiApiKey')) { + availableProviders.push(providerName); + } + break; + case 'anthropic': + if (options.getOption('anthropicApiKey')) { + availableProviders.push(providerName); + } + break; + case 'ollama': + if (options.getOption('ollamaBaseUrl')) { + availableProviders.push(providerName); + } + break; + } + } catch (error) { + // Ignore configuration errors, provider just won't be available + } + } + } + + return availableProviders; } /** @@ -224,9 +253,12 @@ export class AIServiceManager implements IAIServiceManager { if (modelIdentifier.provider && availableProviders.includes(modelIdentifier.provider as ServiceProviders)) { try { - const modifiedOptions = { ...options, model: modelIdentifier.modelId }; - log.info(`[AIServiceManager] Using provider ${modelIdentifier.provider} from model prefix with modifiedOptions.stream: ${modifiedOptions.stream}`); - return await this.services[modelIdentifier.provider as ServiceProviders].generateChatCompletion(messages, modifiedOptions); + const service = this.services[modelIdentifier.provider as ServiceProviders]; + if (service) { + const modifiedOptions = { ...options, model: modelIdentifier.modelId }; + log.info(`[AIServiceManager] Using provider ${modelIdentifier.provider} from model prefix with modifiedOptions.stream: ${modifiedOptions.stream}`); + return await service.generateChatCompletion(messages, modifiedOptions); + } } catch (error) { log.error(`Error with specified provider ${modelIdentifier.provider}: ${error}`); // If the specified provider fails, continue with the fallback providers @@ -240,8 +272,11 @@ export class AIServiceManager implements IAIServiceManager { for (const provider of sortedProviders) { try { - log.info(`[AIServiceManager] Trying provider ${provider} with options.stream: ${options.stream}`); - return await this.services[provider].generateChatCompletion(messages, options); + const service = this.services[provider]; + if (service) { + log.info(`[AIServiceManager] Trying provider ${provider} with options.stream: ${options.stream}`); + return await service.generateChatCompletion(messages, options); + } } catch (error) { log.error(`Error with provider ${provider}: ${error}`); lastError = error as Error; @@ -348,30 +383,49 @@ export class AIServiceManager implements IAIServiceManager { } /** - * Set up embeddings provider using the new configuration system + * Get or create a chat provider on-demand */ - async setupEmbeddingsProvider(): Promise { - try { - const aiEnabled = await isAIEnabled(); - if (!aiEnabled) { - log.info('AI features are disabled'); - return; - } - - // Use the new configuration system - no string parsing! - const enabledProviders = await getEnabledEmbeddingProviders(); - - if (enabledProviders.length === 0) { - log.info('No embedding providers are enabled'); - return; - } - - // Initialize embedding providers - log.info('Embedding providers initialized successfully'); - } catch (error: any) { - log.error(`Error setting up embedding providers: ${error.message}`); - throw error; + private async getOrCreateChatProvider(providerName: ServiceProviders): Promise { + // Return existing provider if already created + if (this.services[providerName]) { + return this.services[providerName]; } + + // Create provider on-demand based on configuration + try { + switch (providerName) { + case 'openai': + const openaiApiKey = await options.getOption('openaiApiKey'); + if (openaiApiKey) { + this.services.openai = new OpenAIService(); + log.info('Created OpenAI chat provider on-demand'); + return this.services.openai; + } + break; + + case 'anthropic': + const anthropicApiKey = await options.getOption('anthropicApiKey'); + if (anthropicApiKey) { + this.services.anthropic = new AnthropicService(); + log.info('Created Anthropic chat provider on-demand'); + return this.services.anthropic; + } + break; + + case 'ollama': + const ollamaBaseUrl = await options.getOption('ollamaBaseUrl'); + if (ollamaBaseUrl) { + this.services.ollama = new OllamaService(); + log.info('Created Ollama chat provider on-demand'); + return this.services.ollama; + } + break; + } + } catch (error: any) { + log.error(`Error creating ${providerName} provider on-demand: ${error.message || 'Unknown error'}`); + } + + return null; } /** @@ -392,9 +446,6 @@ export class AIServiceManager implements IAIServiceManager { // Update provider order from configuration await this.updateProviderOrderAsync(); - // Set up embeddings provider if AI is enabled - await this.setupEmbeddingsProvider(); - // Initialize index service await this.getIndexService().initialize(); @@ -462,7 +513,7 @@ export class AIServiceManager implements IAIServiceManager { try { // Get the default LLM service for context enhancement const provider = this.getPreferredProvider(); - const llmService = this.getService(provider); + const llmService = await this.getService(provider); // Find relevant notes contextNotes = await contextService.findRelevantNotes( @@ -503,25 +554,27 @@ export class AIServiceManager implements IAIServiceManager { /** * Get AI service for the given provider */ - getService(provider?: string): AIService { + async getService(provider?: string): Promise { this.ensureInitialized(); - // If provider is specified, try to use it - if (provider && this.services[provider as ServiceProviders]?.isAvailable()) { - return this.services[provider as ServiceProviders]; - } - - // Otherwise, use the first available provider in the configured order - for (const providerName of this.providerOrder) { - const service = this.services[providerName]; - if (service.isAvailable()) { + // If provider is specified, try to get or create it + if (provider) { + const service = await this.getOrCreateChatProvider(provider as ServiceProviders); + if (service && service.isAvailable()) { return service; } } - // If no provider is available, use first one anyway (it will throw an error) - // This allows us to show a proper error message rather than "provider not found" - return this.services[this.providerOrder[0]]; + // Otherwise, try providers in the configured order + for (const providerName of this.providerOrder) { + const service = await this.getOrCreateChatProvider(providerName); + if (service && service.isAvailable()) { + return service; + } + } + + // If no provider is available, throw a clear error + throw new Error('No AI chat providers are available. Please check your AI settings.'); } /** @@ -550,7 +603,8 @@ export class AIServiceManager implements IAIServiceManager { // Return the first available provider in the order for (const providerName of this.providerOrder) { - if (this.services[providerName].isAvailable()) { + const service = this.services[providerName]; + if (service && service.isAvailable()) { return providerName; } } @@ -634,13 +688,15 @@ export class AIServiceManager implements IAIServiceManager { // Initialize embeddings through index service await indexService.startEmbeddingGeneration(); } else { - log.info('AI features disabled, stopping embeddings'); + log.info('AI features disabled, stopping embeddings and clearing providers'); // Stop embeddings through index service await indexService.stopEmbeddingGeneration(); + // Clear chat providers + this.services = {}; } } else { - // For other AI-related options, just recreate services - this.recreateServices(); + // For other AI-related options, recreate services on-demand + await this.recreateServices(); } } }); @@ -656,8 +712,12 @@ export class AIServiceManager implements IAIServiceManager { // Clear configuration cache first clearConfigurationCache(); - // Recreate all service instances to pick up new configuration - this.recreateServiceInstances(); + // Clear existing chat providers (they will be recreated on-demand) + this.services = {}; + + // Clear embedding providers (they will be recreated on-demand when needed) + const providerManager = await import('./providers/providers.js'); + providerManager.clearAllEmbeddingProviders(); // Update provider order with new configuration await this.updateProviderOrderAsync(); @@ -668,25 +728,6 @@ export class AIServiceManager implements IAIServiceManager { } } - /** - * Recreate service instances to pick up new configuration - */ - private recreateServiceInstances(): void { - try { - log.info('Recreating service instances'); - - // Recreate service instances - this.services = { - openai: new OpenAIService(), - anthropic: new AnthropicService(), - ollama: new OllamaService() - }; - - log.info('Service instances recreated successfully'); - } catch (error) { - log.error(`Error recreating service instances: ${this.handleError(error)}`); - } - } } // Don't create singleton immediately, use a lazy-loading pattern @@ -759,7 +800,7 @@ export default { ); }, // New methods - getService(provider?: string): AIService { + async getService(provider?: string): Promise { return getInstance().getService(provider); }, getPreferredProvider(): string { diff --git a/apps/server/src/services/llm/context/index.ts b/apps/server/src/services/llm/context/index.ts index 258428705..c44eea9cb 100644 --- a/apps/server/src/services/llm/context/index.ts +++ b/apps/server/src/services/llm/context/index.ts @@ -33,7 +33,7 @@ async function getSemanticContext( } // Get an LLM service - const llmService = aiServiceManager.getInstance().getService(); + const llmService = await aiServiceManager.getInstance().getService(); const result = await contextService.processQuery("", llmService, { maxResults: options.maxSimilarNotes || 5, @@ -543,7 +543,7 @@ export class ContextExtractor { try { const { default: aiServiceManager } = await import('../ai_service_manager.js'); const contextService = aiServiceManager.getInstance().getContextService(); - const llmService = aiServiceManager.getInstance().getService(); + const llmService = await aiServiceManager.getInstance().getService(); if (!contextService) { return "Context service not available."; diff --git a/apps/server/src/services/llm/embeddings/init.ts b/apps/server/src/services/llm/embeddings/init.ts index 94188fa69..6ee4afe4b 100644 --- a/apps/server/src/services/llm/embeddings/init.ts +++ b/apps/server/src/services/llm/embeddings/init.ts @@ -45,8 +45,7 @@ export async function initializeEmbeddings() { // Start the embedding system if AI is enabled if (await options.getOptionBool('aiEnabled')) { - // Initialize default embedding providers when AI is enabled - await providerManager.initializeDefaultProviders(); + // Embedding providers will be created on-demand when needed await initEmbeddings(); log.info("Embedding system initialized successfully."); } else { diff --git a/apps/server/src/services/llm/index_service.ts b/apps/server/src/services/llm/index_service.ts index a179431ee..9d118e274 100644 --- a/apps/server/src/services/llm/index_service.ts +++ b/apps/server/src/services/llm/index_service.ts @@ -851,10 +851,6 @@ export class IndexService { throw new Error("AI features must be enabled first"); } - // Re-initialize providers first in case they weren't available when server started - log.info("Re-initializing embedding providers"); - await providerManager.initializeDefaultProviders(); - // Re-initialize if needed if (!this.initialized) { await this.initialize(); @@ -870,6 +866,13 @@ export class IndexService { return; } + // Verify providers are available (this will create them on-demand if needed) + const providers = await providerManager.getEnabledEmbeddingProviders(); + if (providers.length === 0) { + throw new Error("No embedding providers available"); + } + log.info(`Found ${providers.length} embedding providers: ${providers.map(p => p.name).join(', ')}`); + // Setup automatic indexing if enabled if (await options.getOptionBool('embeddingAutoUpdateEnabled')) { this.setupAutomaticIndexing(); diff --git a/apps/server/src/services/llm/interfaces/ai_service_interfaces.ts b/apps/server/src/services/llm/interfaces/ai_service_interfaces.ts index 3126691a4..4130a8d55 100644 --- a/apps/server/src/services/llm/interfaces/ai_service_interfaces.ts +++ b/apps/server/src/services/llm/interfaces/ai_service_interfaces.ts @@ -28,7 +28,7 @@ export interface AIServiceManagerConfig { * Interface for managing AI service providers */ export interface IAIServiceManager { - getService(provider?: string): AIService; + getService(provider?: string): Promise; getAvailableProviders(): string[]; getPreferredProvider(): string; isProviderAvailable(provider: string): boolean; diff --git a/apps/server/src/services/llm/pipeline/stages/context_extraction_stage.ts b/apps/server/src/services/llm/pipeline/stages/context_extraction_stage.ts index 95d7620e2..b1eaa69f9 100644 --- a/apps/server/src/services/llm/pipeline/stages/context_extraction_stage.ts +++ b/apps/server/src/services/llm/pipeline/stages/context_extraction_stage.ts @@ -43,7 +43,7 @@ export class ContextExtractionStage { // Get enhanced context from the context service const contextService = aiServiceManager.getContextService(); - const llmService = aiServiceManager.getService(); + const llmService = await aiServiceManager.getService(); if (contextService) { // Use unified context service to get smart context diff --git a/apps/server/src/services/llm/pipeline/stages/llm_completion_stage.ts b/apps/server/src/services/llm/pipeline/stages/llm_completion_stage.ts index 7dd6984c8..6354e4c59 100644 --- a/apps/server/src/services/llm/pipeline/stages/llm_completion_stage.ts +++ b/apps/server/src/services/llm/pipeline/stages/llm_completion_stage.ts @@ -104,7 +104,7 @@ export class LLMCompletionStage extends BasePipelineStage { + const result: EmbeddingProvider[] = []; + + try { + // Create Ollama provider if embedding base URL is configured + const ollamaEmbeddingBaseUrl = await options.getOption('ollamaEmbeddingBaseUrl'); + if (ollamaEmbeddingBaseUrl) { + const embeddingModel = await options.getOption('ollamaEmbeddingModel'); + + try { + const ollamaProvider = new OllamaEmbeddingProvider({ + model: embeddingModel, + dimension: 768, // Initial value, will be updated during initialization + type: 'float32', + baseUrl: ollamaEmbeddingBaseUrl + }); + + await ollamaProvider.initialize(); + registerEmbeddingProvider(ollamaProvider); + result.push(ollamaProvider); + log.info(`Created Ollama provider on-demand: ${embeddingModel} at ${ollamaEmbeddingBaseUrl}`); + } catch (error: any) { + log.error(`Error creating Ollama embedding provider on-demand: ${error.message || 'Unknown error'}`); + } + } + + // Create OpenAI provider if API key is configured + const openaiApiKey = await options.getOption('openaiApiKey'); + if (openaiApiKey) { + const openaiModel = await options.getOption('openaiEmbeddingModel') || 'text-embedding-3-small'; + const openaiBaseUrl = await options.getOption('openaiBaseUrl') || 'https://api.openai.com/v1'; + + const openaiProvider = new OpenAIEmbeddingProvider({ + model: openaiModel, + dimension: 1536, + type: 'float32', + apiKey: openaiApiKey, + baseUrl: openaiBaseUrl + }); + + registerEmbeddingProvider(openaiProvider); + result.push(openaiProvider); + log.info(`Created OpenAI provider on-demand: ${openaiModel}`); + } + + // Create Voyage provider if API key is configured + const voyageApiKey = await options.getOption('voyageApiKey' as any); + if (voyageApiKey) { + const voyageModel = await options.getOption('voyageEmbeddingModel') || 'voyage-2'; + const voyageBaseUrl = 'https://api.voyageai.com/v1'; + + const voyageProvider = new VoyageEmbeddingProvider({ + model: voyageModel, + dimension: 1024, + type: 'float32', + apiKey: voyageApiKey, + baseUrl: voyageBaseUrl + }); + + registerEmbeddingProvider(voyageProvider); + result.push(voyageProvider); + log.info(`Created Voyage provider on-demand: ${voyageModel}`); + } + + // Always include local provider as fallback + if (!providers.has('local')) { + const localProvider = new SimpleLocalEmbeddingProvider({ + model: 'local', + dimension: 384, + type: 'float32' + }); + registerEmbeddingProvider(localProvider); + result.push(localProvider); + log.info(`Created local provider on-demand as fallback`); + } else { + result.push(providers.get('local')!); + } + + } catch (error: any) { + log.error(`Error creating providers from current options: ${error.message || 'Unknown error'}`); + } + + return result; +} + /** * Get all enabled embedding providers */ @@ -131,31 +219,16 @@ export async function getEnabledEmbeddingProviders(): Promise)) }); - if (result && result.text) { - return result.text; - } - } catch (error) { - log.error(`Error summarizing content: ${error}`); - // Fall through to smart truncation if summarization fails + if (result && result.text) { + return result.text; } + } catch (error) { + log.error(`Error summarizing content: ${error}`); + // Fall through to smart truncation if summarization fails } } From 3a4bb47cc192dde68fe5204354d383c4dcb08532 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Thu, 5 Jun 2025 21:03:15 +0000 Subject: [PATCH 10/18] feat(llm): embeddings work and are created when launching for the first ever time --- .../src/services/llm/embeddings/index.ts | 2 + .../src/services/llm/embeddings/queue.ts | 3 - .../src/services/llm/embeddings/stats.ts | 29 ++--- apps/server/src/services/llm/index_service.ts | 103 +++++++++++++++++- 4 files changed, 113 insertions(+), 24 deletions(-) diff --git a/apps/server/src/services/llm/embeddings/index.ts b/apps/server/src/services/llm/embeddings/index.ts index 2757f8808..c931a1745 100644 --- a/apps/server/src/services/llm/embeddings/index.ts +++ b/apps/server/src/services/llm/embeddings/index.ts @@ -65,6 +65,7 @@ export const { export const { getEmbeddingStats, reprocessAllNotes, + queueNotesForMissingEmbeddings, cleanupEmbeddings } = stats; @@ -107,6 +108,7 @@ export default { // Stats and maintenance getEmbeddingStats: stats.getEmbeddingStats, reprocessAllNotes: stats.reprocessAllNotes, + queueNotesForMissingEmbeddings: stats.queueNotesForMissingEmbeddings, cleanupEmbeddings: stats.cleanupEmbeddings, // Index operations diff --git a/apps/server/src/services/llm/embeddings/queue.ts b/apps/server/src/services/llm/embeddings/queue.ts index 12f915c81..b002513c5 100644 --- a/apps/server/src/services/llm/embeddings/queue.ts +++ b/apps/server/src/services/llm/embeddings/queue.ts @@ -282,8 +282,6 @@ export async function processEmbeddingQueue() { continue; } - // Log that we're starting to process this note - log.info(`Starting embedding generation for note ${noteId}`); // Get note context for embedding const context = await getNoteEmbeddingContext(noteId); @@ -334,7 +332,6 @@ export async function processEmbeddingQueue() { "DELETE FROM embedding_queue WHERE noteId = ?", [noteId] ); - log.info(`Successfully completed embedding processing for note ${noteId}`); // Count as successfully processed processedCount++; diff --git a/apps/server/src/services/llm/embeddings/stats.ts b/apps/server/src/services/llm/embeddings/stats.ts index 6154da368..7fa0d6d82 100644 --- a/apps/server/src/services/llm/embeddings/stats.ts +++ b/apps/server/src/services/llm/embeddings/stats.ts @@ -1,28 +1,13 @@ import sql from "../../../services/sql.js"; import log from "../../../services/log.js"; -import cls from "../../../services/cls.js"; -import { queueNoteForEmbedding } from "./queue.js"; +import indexService from '../index_service.js'; /** * Reprocess all notes to update embeddings + * @deprecated Use indexService.reprocessAllNotes() directly instead */ export async function reprocessAllNotes() { - log.info("Queueing all notes for embedding updates"); - - // Get all non-deleted note IDs - const noteIds = await sql.getColumn( - "SELECT noteId FROM notes WHERE isDeleted = 0" - ); - - log.info(`Adding ${noteIds.length} notes to embedding queue`); - - // Process each note ID within a cls context - for (const noteId of noteIds) { - // Use cls.init to ensure proper context for each operation - await cls.init(async () => { - await queueNoteForEmbedding(noteId as string, 'UPDATE'); - }); - } + return indexService.reprocessAllNotes(); } /** @@ -79,6 +64,14 @@ export async function getEmbeddingStats() { }; } +/** + * Queue notes that don't have embeddings for current provider settings + * @deprecated Use indexService.queueNotesForMissingEmbeddings() directly instead + */ +export async function queueNotesForMissingEmbeddings() { + return indexService.queueNotesForMissingEmbeddings(); +} + /** * Cleanup function to remove stale or unused embeddings */ diff --git a/apps/server/src/services/llm/index_service.ts b/apps/server/src/services/llm/index_service.ts index 9d118e274..08d79dcb1 100644 --- a/apps/server/src/services/llm/index_service.ts +++ b/apps/server/src/services/llm/index_service.ts @@ -12,6 +12,7 @@ import log from "../log.js"; import options from "../options.js"; import becca from "../../becca/becca.js"; +import beccaLoader from "../../becca/becca_loader.js"; import vectorStore from "./embeddings/index.js"; import providerManager from "./providers/providers.js"; import { ContextExtractor } from "./context/index.js"; @@ -378,11 +379,10 @@ export class IndexService { if (!shouldProcessEmbeddings) { // This instance is not configured to process embeddings - log.info("Skipping batch indexing as this instance is not configured to process embeddings"); return false; } - // Process the embedding queue + // Process the embedding queue (batch size is controlled by embeddingBatchSize option) await vectorStore.processEmbeddingQueue(); return true; @@ -879,9 +879,16 @@ export class IndexService { log.info(`Automatic embedding indexing started ${isSyncServer ? 'as sync server' : 'as client'}`); } + // Start background processing of the embedding queue + const { setupEmbeddingBackgroundProcessing } = await import('./embeddings/events.js'); + await setupEmbeddingBackgroundProcessing(); + // Re-initialize event listeners this.setupEventListeners(); + // Queue notes that don't have embeddings for current providers + await this.queueNotesForMissingEmbeddings(); + // Start processing the queue immediately await this.runBatchIndexing(20); @@ -892,6 +899,95 @@ export class IndexService { } } + + + /** + * Queue notes that don't have embeddings for current provider settings + */ + async queueNotesForMissingEmbeddings() { + try { + // Wait for becca to be fully loaded before accessing notes + await beccaLoader.beccaLoaded; + + // Get all non-deleted notes + const allNotes = Object.values(becca.notes).filter(note => !note.isDeleted); + + // Get enabled providers + const providers = await providerManager.getEnabledEmbeddingProviders(); + if (providers.length === 0) { + return; + } + + let queuedCount = 0; + let excludedCount = 0; + + // Process notes in batches to avoid overwhelming the system + const batchSize = 100; + for (let i = 0; i < allNotes.length; i += batchSize) { + const batch = allNotes.slice(i, i + batchSize); + + for (const note of batch) { + try { + // Skip notes excluded from AI + if (isNoteExcludedFromAI(note)) { + excludedCount++; + continue; + } + + // Check if note needs embeddings for any enabled provider + let needsEmbedding = false; + + for (const provider of providers) { + const config = provider.getConfig(); + const existingEmbedding = await vectorStore.getEmbeddingForNote( + note.noteId, + provider.name, + config.model + ); + + if (!existingEmbedding) { + needsEmbedding = true; + break; + } + } + + if (needsEmbedding) { + await vectorStore.queueNoteForEmbedding(note.noteId, 'UPDATE'); + queuedCount++; + } + } catch (error: any) { + log.error(`Error checking embeddings for note ${note.noteId}: ${error.message || 'Unknown error'}`); + } + } + + } + } catch (error: any) { + log.error(`Error queuing notes for missing embeddings: ${error.message || 'Unknown error'}`); + } + } + + /** + * Reprocess all notes to update embeddings + */ + async reprocessAllNotes() { + if (!this.initialized) { + await this.initialize(); + } + + try { + // Get all non-deleted note IDs + const noteIds = await sql.getColumn("SELECT noteId FROM notes WHERE isDeleted = 0"); + + // Process each note ID + for (const noteId of noteIds) { + await vectorStore.queueNoteForEmbedding(noteId as string, 'UPDATE'); + } + } catch (error: any) { + log.error(`Error reprocessing all notes: ${error.message || 'Unknown error'}`); + throw error; + } + } + /** * Stop embedding generation (called when AI is disabled) */ @@ -907,7 +1003,8 @@ export class IndexService { } // Stop the background processing from embeddings/events.ts - vectorStore.stopEmbeddingBackgroundProcessing(); + const { stopEmbeddingBackgroundProcessing } = await import('./embeddings/events.js'); + stopEmbeddingBackgroundProcessing(); // Clear all embedding providers to clean up resources providerManager.clearAllEmbeddingProviders(); From c26b74495c3b7f5062bc1d38dc0b3e0c2e4ca168 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Thu, 5 Jun 2025 22:34:20 +0000 Subject: [PATCH 11/18] feat(llm): remove LLM deprecated functions --- apps/server/src/routes/api/embeddings.ts | 2 +- .../src/services/llm/ai_service_manager.ts | 10 ++--- .../services/llm/chat/rest_chat_service.ts | 4 +- .../llm/config/configuration_helpers.ts | 44 ------------------- .../llm/context/modules/provider_manager.ts | 8 ++-- .../llm/context/services/context_service.ts | 4 +- .../context/services/vector_search_service.ts | 4 +- .../src/services/llm/embeddings/index.ts | 4 -- .../src/services/llm/embeddings/stats.ts | 17 ------- .../src/services/llm/embeddings/storage.ts | 7 +-- apps/server/src/services/llm/index_service.ts | 4 +- .../llm/interfaces/ai_service_interfaces.ts | 2 +- .../semantic_context_extraction_stage.ts | 2 +- .../src/services/llm/providers/providers.ts | 14 +----- 14 files changed, 25 insertions(+), 101 deletions(-) diff --git a/apps/server/src/routes/api/embeddings.ts b/apps/server/src/routes/api/embeddings.ts index 8fd9b475a..226231ea2 100644 --- a/apps/server/src/routes/api/embeddings.ts +++ b/apps/server/src/routes/api/embeddings.ts @@ -408,7 +408,7 @@ async function reprocessAllNotes(req: Request, res: Response) { try { // Wrap the operation in cls.init to ensure proper context cls.init(async () => { - await vectorStore.reprocessAllNotes(); + await indexService.reprocessAllNotes(); log.info("Embedding reprocessing completed successfully"); }); } catch (error: any) { diff --git a/apps/server/src/services/llm/ai_service_manager.ts b/apps/server/src/services/llm/ai_service_manager.ts index f054dff57..805034072 100644 --- a/apps/server/src/services/llm/ai_service_manager.ts +++ b/apps/server/src/services/llm/ai_service_manager.ts @@ -512,7 +512,7 @@ export class AIServiceManager implements IAIServiceManager { if (!contextNotes || contextNotes.length === 0) { try { // Get the default LLM service for context enhancement - const provider = this.getPreferredProvider(); + const provider = this.getSelectedProvider(); const llmService = await this.getService(provider); // Find relevant notes @@ -596,9 +596,9 @@ export class AIServiceManager implements IAIServiceManager { } /** - * Get the preferred provider based on configuration (sync version for compatibility) + * Get the selected provider based on configuration (sync version for compatibility) */ - getPreferredProvider(): string { + getSelectedProvider(): string { this.ensureInitialized(); // Return the first available provider in the order @@ -803,8 +803,8 @@ export default { async getService(provider?: string): Promise { return getInstance().getService(provider); }, - getPreferredProvider(): string { - return getInstance().getPreferredProvider(); + getSelectedProvider(): string { + return getInstance().getSelectedProvider(); }, isProviderAvailable(provider: string): boolean { return getInstance().isProviderAvailable(provider); diff --git a/apps/server/src/services/llm/chat/rest_chat_service.ts b/apps/server/src/services/llm/chat/rest_chat_service.ts index 1ad3d7a22..4176eaffe 100644 --- a/apps/server/src/services/llm/chat/rest_chat_service.ts +++ b/apps/server/src/services/llm/chat/rest_chat_service.ts @@ -14,7 +14,7 @@ import type { LLMStreamMessage } from "../interfaces/chat_ws_messages.js"; import chatStorageService from '../chat_storage_service.js'; import { isAIEnabled, - getFirstValidModelConfig, + getSelectedModelConfig, } from '../config/configuration_helpers.js'; /** @@ -419,7 +419,7 @@ class RestChatService { */ async getPreferredModel(): Promise { try { - const validConfig = await getFirstValidModelConfig(); + const validConfig = await getSelectedModelConfig(); if (!validConfig) { log.error('No valid AI model configuration found'); return undefined; diff --git a/apps/server/src/services/llm/config/configuration_helpers.ts b/apps/server/src/services/llm/config/configuration_helpers.ts index 286716d3c..2635cc35f 100644 --- a/apps/server/src/services/llm/config/configuration_helpers.ts +++ b/apps/server/src/services/llm/config/configuration_helpers.ts @@ -150,47 +150,3 @@ export async function getSelectedModelConfig(): Promise<{ model: string; provide return await getValidModelConfig(selectedProvider); } -// Legacy support functions - these maintain backwards compatibility but now use single provider logic -/** - * @deprecated Use getSelectedProvider() instead - */ -export async function getProviderPrecedence(): Promise { - const selected = await getSelectedProvider(); - return selected ? [selected] : []; -} - -/** - * @deprecated Use getSelectedProvider() instead - */ -export async function getPreferredProvider(): Promise { - return await getSelectedProvider(); -} - -/** - * @deprecated Use getSelectedEmbeddingProvider() instead - */ -export async function getEmbeddingProviderPrecedence(): Promise { - const selected = await getSelectedEmbeddingProvider(); - return selected ? [selected] : []; -} - -/** - * @deprecated Use getSelectedEmbeddingProvider() instead - */ -export async function getPreferredEmbeddingProvider(): Promise { - return await getSelectedEmbeddingProvider(); -} - -/** - * @deprecated Use getAvailableSelectedProvider() instead - */ -export async function getFirstAvailableProvider(): Promise { - return await getAvailableSelectedProvider(); -} - -/** - * @deprecated Use getSelectedModelConfig() instead - */ -export async function getFirstValidModelConfig(): Promise<{ model: string; provider: ProviderType } | null> { - return await getSelectedModelConfig(); -} \ No newline at end of file diff --git a/apps/server/src/services/llm/context/modules/provider_manager.ts b/apps/server/src/services/llm/context/modules/provider_manager.ts index 6af8ac991..56c6437c0 100644 --- a/apps/server/src/services/llm/context/modules/provider_manager.ts +++ b/apps/server/src/services/llm/context/modules/provider_manager.ts @@ -1,6 +1,6 @@ import log from '../../../log.js'; import { getEmbeddingProvider, getEnabledEmbeddingProviders } from '../../providers/providers.js'; -import { getSelectedEmbeddingProvider } from '../../config/configuration_helpers.js'; +import { getSelectedEmbeddingProvider as getSelectedEmbeddingProviderName } from '../../config/configuration_helpers.js'; /** * Manages embedding providers for context services @@ -12,10 +12,10 @@ export class ProviderManager { * * @returns The selected embedding provider or null if none available */ - async getPreferredEmbeddingProvider(): Promise { + async getSelectedEmbeddingProvider(): Promise { try { // Get the selected embedding provider - const selectedProvider = await getSelectedEmbeddingProvider(); + const selectedProvider = await getSelectedEmbeddingProviderName(); if (selectedProvider) { const provider = await getEmbeddingProvider(selectedProvider); @@ -51,7 +51,7 @@ export class ProviderManager { async generateQueryEmbedding(query: string): Promise { try { // Get the preferred embedding provider - const provider = await this.getPreferredEmbeddingProvider(); + const provider = await this.getSelectedEmbeddingProvider(); if (!provider) { log.error('No embedding provider available'); return null; diff --git a/apps/server/src/services/llm/context/services/context_service.ts b/apps/server/src/services/llm/context/services/context_service.ts index a227c3936..b4d4fd613 100644 --- a/apps/server/src/services/llm/context/services/context_service.ts +++ b/apps/server/src/services/llm/context/services/context_service.ts @@ -58,7 +58,7 @@ export class ContextService { this.initPromise = (async () => { try { // Initialize provider - const provider = await providerManager.getPreferredEmbeddingProvider(); + const provider = await providerManager.getSelectedEmbeddingProvider(); if (!provider) { throw new Error(`No embedding provider available. Could not initialize context service.`); } @@ -224,7 +224,7 @@ export class ContextService { log.info(`Final combined results: ${relevantNotes.length} relevant notes`); // Step 4: Build context from the notes - const provider = await providerManager.getPreferredEmbeddingProvider(); + const provider = await providerManager.getSelectedEmbeddingProvider(); const providerId = provider?.name || 'default'; const context = await contextFormatter.buildContextFromNotes( diff --git a/apps/server/src/services/llm/context/services/vector_search_service.ts b/apps/server/src/services/llm/context/services/vector_search_service.ts index 480ba05bd..98c7b993c 100644 --- a/apps/server/src/services/llm/context/services/vector_search_service.ts +++ b/apps/server/src/services/llm/context/services/vector_search_service.ts @@ -79,7 +79,7 @@ export class VectorSearchService { } // Get provider information - const provider = await providerManager.getPreferredEmbeddingProvider(); + const provider = await providerManager.getSelectedEmbeddingProvider(); if (!provider) { log.error('No embedding provider available'); return []; @@ -280,7 +280,7 @@ export class VectorSearchService { } // Get provider information - const provider = await providerManager.getPreferredEmbeddingProvider(); + const provider = await providerManager.getSelectedEmbeddingProvider(); if (!provider) { log.error('No embedding provider available'); return []; diff --git a/apps/server/src/services/llm/embeddings/index.ts b/apps/server/src/services/llm/embeddings/index.ts index c931a1745..c4be44a2e 100644 --- a/apps/server/src/services/llm/embeddings/index.ts +++ b/apps/server/src/services/llm/embeddings/index.ts @@ -64,8 +64,6 @@ export const { export const { getEmbeddingStats, - reprocessAllNotes, - queueNotesForMissingEmbeddings, cleanupEmbeddings } = stats; @@ -107,8 +105,6 @@ export default { // Stats and maintenance getEmbeddingStats: stats.getEmbeddingStats, - reprocessAllNotes: stats.reprocessAllNotes, - queueNotesForMissingEmbeddings: stats.queueNotesForMissingEmbeddings, cleanupEmbeddings: stats.cleanupEmbeddings, // Index operations diff --git a/apps/server/src/services/llm/embeddings/stats.ts b/apps/server/src/services/llm/embeddings/stats.ts index 7fa0d6d82..a8b594723 100644 --- a/apps/server/src/services/llm/embeddings/stats.ts +++ b/apps/server/src/services/llm/embeddings/stats.ts @@ -1,14 +1,5 @@ import sql from "../../../services/sql.js"; import log from "../../../services/log.js"; -import indexService from '../index_service.js'; - -/** - * Reprocess all notes to update embeddings - * @deprecated Use indexService.reprocessAllNotes() directly instead - */ -export async function reprocessAllNotes() { - return indexService.reprocessAllNotes(); -} /** * Get current embedding statistics @@ -64,14 +55,6 @@ export async function getEmbeddingStats() { }; } -/** - * Queue notes that don't have embeddings for current provider settings - * @deprecated Use indexService.queueNotesForMissingEmbeddings() directly instead - */ -export async function queueNotesForMissingEmbeddings() { - return indexService.queueNotesForMissingEmbeddings(); -} - /** * Cleanup function to remove stale or unused embeddings */ diff --git a/apps/server/src/services/llm/embeddings/storage.ts b/apps/server/src/services/llm/embeddings/storage.ts index ac096071f..bbd1eba9c 100644 --- a/apps/server/src/services/llm/embeddings/storage.ts +++ b/apps/server/src/services/llm/embeddings/storage.ts @@ -11,7 +11,7 @@ import { SEARCH_CONSTANTS } from '../constants/search_constants.js'; import type { NoteEmbeddingContext } from "./embeddings_interface.js"; import becca from "../../../becca/becca.js"; import { isNoteExcludedFromAIById } from "../utils/ai_exclusion_utils.js"; -import { getEmbeddingProviderPrecedence } from '../config/configuration_helpers.js'; +import { getSelectedEmbeddingProvider } from '../config/configuration_helpers.js'; interface Similarity { noteId: string; @@ -277,9 +277,10 @@ export async function findSimilarNotes( log.info('No embeddings found for specified provider, trying fallback providers...'); // Use the new configuration system - no string parsing! - const preferredProviders = await getEmbeddingProviderPrecedence(); + const selectedProvider = await getSelectedEmbeddingProvider(); + const preferredProviders = selectedProvider ? [selectedProvider] : []; - log.info(`Using provider precedence: ${preferredProviders.join(', ')}`); + log.info(`Using selected provider: ${selectedProvider || 'none'}`); // Try providers in precedence order for (const provider of preferredProviders) { diff --git a/apps/server/src/services/llm/index_service.ts b/apps/server/src/services/llm/index_service.ts index 08d79dcb1..957cfa7d7 100644 --- a/apps/server/src/services/llm/index_service.ts +++ b/apps/server/src/services/llm/index_service.ts @@ -266,7 +266,7 @@ export class IndexService { this.indexRebuildTotal = totalNotes; log.info("No embeddings found, starting full embedding generation first"); - await vectorStore.reprocessAllNotes(); + await this.reprocessAllNotes(); log.info("Full embedding generation initiated"); } else { // For index rebuild, use the number of embeddings as the total @@ -293,7 +293,7 @@ export class IndexService { // Only start indexing if we're below 90% completion or if embeddings exist but need optimization if (stats.percentComplete < 90) { log.info("Embedding coverage below 90%, starting full embedding generation"); - await vectorStore.reprocessAllNotes(); + await this.reprocessAllNotes(); log.info("Full embedding generation initiated"); } else { log.info(`Embedding coverage at ${stats.percentComplete}%, starting index optimization`); diff --git a/apps/server/src/services/llm/interfaces/ai_service_interfaces.ts b/apps/server/src/services/llm/interfaces/ai_service_interfaces.ts index 4130a8d55..52736cbd5 100644 --- a/apps/server/src/services/llm/interfaces/ai_service_interfaces.ts +++ b/apps/server/src/services/llm/interfaces/ai_service_interfaces.ts @@ -30,7 +30,7 @@ export interface AIServiceManagerConfig { export interface IAIServiceManager { getService(provider?: string): Promise; getAvailableProviders(): string[]; - getPreferredProvider(): string; + getSelectedProvider(): string; isProviderAvailable(provider: string): boolean; getProviderMetadata(provider: string): ProviderMetadata | null; getAIEnabled(): boolean; diff --git a/apps/server/src/services/llm/pipeline/stages/semantic_context_extraction_stage.ts b/apps/server/src/services/llm/pipeline/stages/semantic_context_extraction_stage.ts index bf2cc8fd7..139510663 100644 --- a/apps/server/src/services/llm/pipeline/stages/semantic_context_extraction_stage.ts +++ b/apps/server/src/services/llm/pipeline/stages/semantic_context_extraction_stage.ts @@ -50,7 +50,7 @@ export class SemanticContextExtractionStage extends BasePipelineStage Date: Fri, 6 Jun 2025 19:22:39 +0000 Subject: [PATCH 12/18] feat(llm): have OpenAI provider not require API keys (for endpoints like LM Studio) --- .../client/src/widgets/llm_chat/validation.ts | 2 +- apps/server/src/routes/api/openai.ts | 11 +++---- .../services/llm/providers/openai_service.ts | 6 ++-- .../src/services/llm/providers/providers.ts | 30 ++++++++++++------- 4 files changed, 30 insertions(+), 19 deletions(-) diff --git a/apps/client/src/widgets/llm_chat/validation.ts b/apps/client/src/widgets/llm_chat/validation.ts index e39b07012..94977f63a 100644 --- a/apps/client/src/widgets/llm_chat/validation.ts +++ b/apps/client/src/widgets/llm_chat/validation.ts @@ -44,7 +44,7 @@ export async function validateEmbeddingProviders(validationWarning: HTMLElement) // Check OpenAI configuration const apiKey = options.get('openaiApiKey'); if (!apiKey) { - configIssues.push(`OpenAI API key is missing`); + configIssues.push(`OpenAI API key is missing (optional for OpenAI-compatible endpoints)`); } } else if (provider === 'anthropic') { // Check Anthropic configuration diff --git a/apps/server/src/routes/api/openai.ts b/apps/server/src/routes/api/openai.ts index ced03ce04..84efad2ca 100644 --- a/apps/server/src/routes/api/openai.ts +++ b/apps/server/src/routes/api/openai.ts @@ -66,12 +66,13 @@ async function listModels(req: Request, res: Response) { const apiKey = await options.getOption('openaiApiKey'); if (!apiKey) { - throw new Error('OpenAI API key is not configured'); + // Log warning but don't throw - some OpenAI-compatible endpoints don't require API keys + log.info('OpenAI API key is not configured when listing models. This may cause issues with official OpenAI endpoints.'); } - // Initialize OpenAI client with the API key and base URL + // Initialize OpenAI client with the API key (or empty string) and base URL const openai = new OpenAI({ - apiKey, + apiKey: apiKey || '', // Default to empty string if no API key baseURL: openaiBaseUrl }); @@ -84,9 +85,9 @@ async function listModels(req: Request, res: Response) { // Include all models as chat models, without filtering by specific model names // This allows models from providers like OpenRouter to be displayed const chatModels = allModels - .filter((model) => + .filter((model) => // Exclude models that are explicitly for embeddings - !model.id.includes('embedding') && + !model.id.includes('embedding') && !model.id.includes('embed') ) .map((model) => ({ diff --git a/apps/server/src/services/llm/providers/openai_service.ts b/apps/server/src/services/llm/providers/openai_service.ts index 09d58498a..411f512f7 100644 --- a/apps/server/src/services/llm/providers/openai_service.ts +++ b/apps/server/src/services/llm/providers/openai_service.ts @@ -14,7 +14,9 @@ export class OpenAIService extends BaseAIService { } override isAvailable(): boolean { - return super.isAvailable() && !!options.getOption('openaiApiKey'); + // Make API key optional to support OpenAI-compatible endpoints that don't require authentication + // The provider is considered available as long as the parent checks pass + return super.isAvailable(); } private getClient(apiKey: string, baseUrl?: string): OpenAI { @@ -29,7 +31,7 @@ export class OpenAIService extends BaseAIService { async generateChatCompletion(messages: Message[], opts: ChatCompletionOptions = {}): Promise { if (!this.isAvailable()) { - throw new Error('OpenAI service is not available. Check API key and AI settings.'); + throw new Error('OpenAI service is not available. Check AI settings.'); } // Get provider-specific options from the central provider manager diff --git a/apps/server/src/services/llm/providers/providers.ts b/apps/server/src/services/llm/providers/providers.ts index 2fe64735a..fd7c603fd 100644 --- a/apps/server/src/services/llm/providers/providers.ts +++ b/apps/server/src/services/llm/providers/providers.ts @@ -134,7 +134,7 @@ export async function createProvidersFromCurrentOptions(): Promise Date: Fri, 6 Jun 2025 20:11:33 +0000 Subject: [PATCH 13/18] feat(llm): for sure overcomplicate what should be a very simple thing --- .../src/services/llm/embeddings/init.ts | 16 +- apps/server/src/services/llm/index_service.ts | 15 +- .../src/services/llm/provider_validation.ts | 330 ++++++++++++++++++ 3 files changed, 356 insertions(+), 5 deletions(-) create mode 100644 apps/server/src/services/llm/provider_validation.ts diff --git a/apps/server/src/services/llm/embeddings/init.ts b/apps/server/src/services/llm/embeddings/init.ts index 6ee4afe4b..75d664e43 100644 --- a/apps/server/src/services/llm/embeddings/init.ts +++ b/apps/server/src/services/llm/embeddings/init.ts @@ -4,6 +4,7 @@ import { initEmbeddings } from "./index.js"; import providerManager from "../providers/providers.js"; import sqlInit from "../../sql_init.js"; import sql from "../../sql.js"; +import { validateProviders, logValidationResults, hasWorkingEmbeddingProviders } from "../provider_validation.js"; /** * Reset any stuck embedding queue items that were left in processing state @@ -45,9 +46,18 @@ export async function initializeEmbeddings() { // Start the embedding system if AI is enabled if (await options.getOptionBool('aiEnabled')) { - // Embedding providers will be created on-demand when needed - await initEmbeddings(); - log.info("Embedding system initialized successfully."); + // Validate providers before starting the embedding system + log.info("Validating AI providers before starting embedding system..."); + const validation = await validateProviders(); + logValidationResults(validation); + + if (await hasWorkingEmbeddingProviders()) { + // Embedding providers will be created on-demand when needed + await initEmbeddings(); + log.info("Embedding system initialized successfully."); + } else { + log.info("Embedding system not started: No working embedding providers found. Please configure at least one AI provider (OpenAI, Ollama, or Voyage) to use embedding features."); + } } else { log.info("Embedding system disabled (AI features are turned off)."); } diff --git a/apps/server/src/services/llm/index_service.ts b/apps/server/src/services/llm/index_service.ts index 957cfa7d7..7786a2cc7 100644 --- a/apps/server/src/services/llm/index_service.ts +++ b/apps/server/src/services/llm/index_service.ts @@ -22,6 +22,7 @@ import sqlInit from "../sql_init.js"; import { CONTEXT_PROMPTS } from './constants/llm_prompt_constants.js'; import { SEARCH_CONSTANTS } from './constants/search_constants.js'; import { isNoteExcludedFromAI } from "./utils/ai_exclusion_utils.js"; +import { hasWorkingEmbeddingProviders } from "./provider_validation.js"; export class IndexService { private initialized = false; @@ -61,9 +62,15 @@ export class IndexService { } // Check if embedding system is ready + if (!(await hasWorkingEmbeddingProviders())) { + log.info("Index service: No working embedding providers available, skipping initialization"); + return; + } + const providers = await providerManager.getEnabledEmbeddingProviders(); if (!providers || providers.length === 0) { - throw new Error("No embedding providers available"); + log.info("Index service: No enabled embedding providers, skipping initialization"); + return; } // Check if this instance should process embeddings @@ -866,7 +873,11 @@ export class IndexService { return; } - // Verify providers are available (this will create them on-demand if needed) + // Verify providers are available + if (!(await hasWorkingEmbeddingProviders())) { + throw new Error("No working embedding providers available"); + } + const providers = await providerManager.getEnabledEmbeddingProviders(); if (providers.length === 0) { throw new Error("No embedding providers available"); diff --git a/apps/server/src/services/llm/provider_validation.ts b/apps/server/src/services/llm/provider_validation.ts new file mode 100644 index 000000000..1cab7bb21 --- /dev/null +++ b/apps/server/src/services/llm/provider_validation.ts @@ -0,0 +1,330 @@ +/** + * Provider Validation Service + * + * Validates AI provider configurations before initializing the embedding system. + * This prevents startup errors when AI is enabled but providers are misconfigured. + */ + +import log from "../log.js"; +import options from "../options.js"; +import type { EmbeddingProvider } from "./embeddings/embeddings_interface.js"; + +export interface ProviderValidationResult { + hasValidProviders: boolean; + validEmbeddingProviders: EmbeddingProvider[]; + validChatProviders: string[]; + errors: string[]; + warnings: string[]; +} + +/** + * Validate all available providers without throwing errors + */ +export async function validateProviders(): Promise { + const result: ProviderValidationResult = { + hasValidProviders: false, + validEmbeddingProviders: [], + validChatProviders: [], + errors: [], + warnings: [] + }; + + try { + // Check if AI is enabled + const aiEnabled = await options.getOptionBool('aiEnabled'); + if (!aiEnabled) { + result.warnings.push("AI features are disabled"); + return result; + } + + // Validate embedding providers + await validateEmbeddingProviders(result); + + // Validate chat providers + await validateChatProviders(result); + + // Determine if we have any valid providers + result.hasValidProviders = result.validEmbeddingProviders.length > 0 || result.validChatProviders.length > 0; + + if (!result.hasValidProviders) { + result.errors.push("No valid AI providers are configured"); + } + + } catch (error: any) { + result.errors.push(`Error during provider validation: ${error.message || 'Unknown error'}`); + } + + return result; +} + +/** + * Validate embedding providers + */ +async function validateEmbeddingProviders(result: ProviderValidationResult): Promise { + try { + // Import provider classes and check configurations + const { OpenAIEmbeddingProvider } = await import("./embeddings/providers/openai.js"); + const { OllamaEmbeddingProvider } = await import("./embeddings/providers/ollama.js"); + const { VoyageEmbeddingProvider } = await import("./embeddings/providers/voyage.js"); + + // Check OpenAI embedding provider + await validateOpenAIEmbeddingProvider(result, OpenAIEmbeddingProvider); + + // Check Ollama embedding provider + await validateOllamaEmbeddingProvider(result, OllamaEmbeddingProvider); + + // Check Voyage embedding provider + await validateVoyageEmbeddingProvider(result, VoyageEmbeddingProvider); + + // Local provider is always available as fallback + await validateLocalEmbeddingProvider(result); + + } catch (error: any) { + result.errors.push(`Error validating embedding providers: ${error.message || 'Unknown error'}`); + } +} + +/** + * Validate chat providers + */ +async function validateChatProviders(result: ProviderValidationResult): Promise { + try { + // Check OpenAI chat provider + const openaiApiKey = await options.getOption('openaiApiKey'); + const openaiBaseUrl = await options.getOption('openaiBaseUrl'); + + if (openaiApiKey || openaiBaseUrl) { + if (!openaiApiKey && !openaiBaseUrl) { + result.warnings.push("OpenAI chat provider: No API key or base URL configured"); + } else if (!openaiApiKey) { + result.warnings.push("OpenAI chat provider: No API key configured (may work with compatible endpoints)"); + result.validChatProviders.push('openai'); + } else { + result.validChatProviders.push('openai'); + } + } + + // Check Anthropic chat provider + const anthropicApiKey = await options.getOption('anthropicApiKey'); + if (anthropicApiKey) { + result.validChatProviders.push('anthropic'); + } else { + result.warnings.push("Anthropic chat provider: No API key configured"); + } + + // Check Ollama chat provider + const ollamaBaseUrl = await options.getOption('ollamaBaseUrl'); + if (ollamaBaseUrl) { + result.validChatProviders.push('ollama'); + } else { + result.warnings.push("Ollama chat provider: No base URL configured"); + } + + } catch (error: any) { + result.errors.push(`Error validating chat providers: ${error.message || 'Unknown error'}`); + } +} + +/** + * Validate OpenAI embedding provider + */ +async function validateOpenAIEmbeddingProvider( + result: ProviderValidationResult, + OpenAIEmbeddingProvider: any +): Promise { + try { + const openaiApiKey = await options.getOption('openaiApiKey'); + const openaiBaseUrl = await options.getOption('openaiBaseUrl'); + + if (openaiApiKey || openaiBaseUrl) { + const openaiModel = await options.getOption('openaiEmbeddingModel'); + const finalBaseUrl = openaiBaseUrl || 'https://api.openai.com/v1'; + + if (!openaiApiKey) { + result.warnings.push("OpenAI embedding provider: No API key configured (may work with compatible endpoints)"); + } + + const provider = new OpenAIEmbeddingProvider({ + model: openaiModel, + dimension: 1536, + type: 'float32', + apiKey: openaiApiKey || '', + baseUrl: finalBaseUrl + }); + + result.validEmbeddingProviders.push(provider); + log.info(`Validated OpenAI embedding provider: ${openaiModel} at ${finalBaseUrl}`); + } else { + result.warnings.push("OpenAI embedding provider: No API key or base URL configured"); + } + } catch (error: any) { + result.errors.push(`OpenAI embedding provider validation failed: ${error.message || 'Unknown error'}`); + } +} + +/** + * Validate Ollama embedding provider + */ +async function validateOllamaEmbeddingProvider( + result: ProviderValidationResult, + OllamaEmbeddingProvider: any +): Promise { + try { + const ollamaEmbeddingBaseUrl = await options.getOption('ollamaEmbeddingBaseUrl'); + + if (ollamaEmbeddingBaseUrl) { + const embeddingModel = await options.getOption('ollamaEmbeddingModel'); + + try { + const provider = new OllamaEmbeddingProvider({ + model: embeddingModel, + dimension: 768, + type: 'float32', + baseUrl: ollamaEmbeddingBaseUrl + }); + + // Try to initialize to validate connection + await provider.initialize(); + result.validEmbeddingProviders.push(provider); + log.info(`Validated Ollama embedding provider: ${embeddingModel} at ${ollamaEmbeddingBaseUrl}`); + } catch (error: any) { + result.warnings.push(`Ollama embedding provider initialization failed: ${error.message || 'Unknown error'}`); + } + } else { + result.warnings.push("Ollama embedding provider: No base URL configured"); + } + } catch (error: any) { + result.errors.push(`Ollama embedding provider validation failed: ${error.message || 'Unknown error'}`); + } +} + +/** + * Validate Voyage embedding provider + */ +async function validateVoyageEmbeddingProvider( + result: ProviderValidationResult, + VoyageEmbeddingProvider: any +): Promise { + try { + const voyageApiKey = await options.getOption('voyageApiKey' as any); + + if (voyageApiKey) { + const voyageModel = await options.getOption('voyageEmbeddingModel') || 'voyage-2'; + + const provider = new VoyageEmbeddingProvider({ + model: voyageModel, + dimension: 1024, + type: 'float32', + apiKey: voyageApiKey, + baseUrl: 'https://api.voyageai.com/v1' + }); + + result.validEmbeddingProviders.push(provider); + log.info(`Validated Voyage embedding provider: ${voyageModel}`); + } else { + result.warnings.push("Voyage embedding provider: No API key configured"); + } + } catch (error: any) { + result.errors.push(`Voyage embedding provider validation failed: ${error.message || 'Unknown error'}`); + } +} + +/** + * Validate local embedding provider (always available as fallback) + */ +async function validateLocalEmbeddingProvider(result: ProviderValidationResult): Promise { + try { + // Simple local embedding provider implementation + class SimpleLocalEmbeddingProvider { + name = "local"; + config = { + model: 'local', + dimension: 384, + type: 'float32' as const + }; + + getConfig() { + return this.config; + } + + getNormalizationStatus() { + return 0; // NormalizationStatus.NEVER + } + + async generateEmbeddings(text: string): Promise { + const result = new Float32Array(this.config.dimension); + for (let i = 0; i < result.length; i++) { + const charSum = Array.from(text).reduce((sum, char, idx) => + sum + char.charCodeAt(0) * Math.sin(idx * 0.1), 0); + result[i] = Math.sin(i * 0.1 + charSum * 0.01); + } + return result; + } + + async generateBatchEmbeddings(texts: string[]): Promise { + return Promise.all(texts.map(text => this.generateEmbeddings(text))); + } + + async generateNoteEmbeddings(context: any): Promise { + const text = (context.title || "") + " " + (context.content || ""); + return this.generateEmbeddings(text); + } + + async generateBatchNoteEmbeddings(contexts: any[]): Promise { + return Promise.all(contexts.map(context => this.generateNoteEmbeddings(context))); + } + } + + const localProvider = new SimpleLocalEmbeddingProvider(); + result.validEmbeddingProviders.push(localProvider as any); + log.info("Validated local embedding provider as fallback"); + } catch (error: any) { + result.errors.push(`Local embedding provider validation failed: ${error.message || 'Unknown error'}`); + } +} + +/** + * Check if any working providers are available for embeddings + */ +export async function hasWorkingEmbeddingProviders(): Promise { + const validation = await validateProviders(); + return validation.validEmbeddingProviders.length > 0; +} + +/** + * Check if any working providers are available for chat + */ +export async function hasWorkingChatProviders(): Promise { + const validation = await validateProviders(); + return validation.validChatProviders.length > 0; +} + +/** + * Get only the working embedding providers + */ +export async function getWorkingEmbeddingProviders(): Promise { + const validation = await validateProviders(); + return validation.validEmbeddingProviders; +} + +/** + * Log validation results in a user-friendly way + */ +export function logValidationResults(validation: ProviderValidationResult): void { + if (validation.hasValidProviders) { + log.info(`AI provider validation passed: ${validation.validEmbeddingProviders.length} embedding providers, ${validation.validChatProviders.length} chat providers`); + + if (validation.validEmbeddingProviders.length > 0) { + log.info(`Working embedding providers: ${validation.validEmbeddingProviders.map(p => p.name).join(', ')}`); + } + + if (validation.validChatProviders.length > 0) { + log.info(`Working chat providers: ${validation.validChatProviders.join(', ')}`); + } + } else { + log.info("AI provider validation failed: No working providers found"); + } + + validation.warnings.forEach(warning => log.info(`Provider validation: ${warning}`)); + validation.errors.forEach(error => log.error(`Provider validation: ${error}`)); +} \ No newline at end of file From 20ec294774199a5d25ad3146bc130b2e5b2450ea Mon Sep 17 00:00:00 2001 From: perf3ct Date: Fri, 6 Jun 2025 20:30:24 +0000 Subject: [PATCH 14/18] feat(llm): still work on decomplicating provider creation --- .../src/services/llm/ai_service_manager.ts | 73 +++-- apps/server/src/services/llm/index_service.ts | 159 +++-------- .../src/services/llm/provider_validation.ts | 265 ++++-------------- .../src/services/llm/providers/providers.ts | 209 +++++++------- 4 files changed, 262 insertions(+), 444 deletions(-) diff --git a/apps/server/src/services/llm/ai_service_manager.ts b/apps/server/src/services/llm/ai_service_manager.ts index 805034072..222f91dfd 100644 --- a/apps/server/src/services/llm/ai_service_manager.ts +++ b/apps/server/src/services/llm/ai_service_manager.ts @@ -267,12 +267,23 @@ export class AIServiceManager implements IAIServiceManager { // If not a provider prefix, treat the entire string as a model name and continue with normal provider selection } - // Try each provider in order until one succeeds + // If user has a specific provider selected, try only that one and fail fast + if (this.providerOrder.length === 1 && sortedProviders.length === 1) { + const selectedProvider = sortedProviders[0]; + const service = await this.getOrCreateChatProvider(selectedProvider); + if (!service) { + throw new Error(`Failed to create selected chat provider: ${selectedProvider}. Please check your configuration.`); + } + log.info(`[AIServiceManager] Using selected provider ${selectedProvider} with options.stream: ${options.stream}`); + return await service.generateChatCompletion(messages, options); + } + + // If no specific provider selected, try each provider in order until one succeeds let lastError: Error | null = null; for (const provider of sortedProviders) { try { - const service = this.services[provider]; + const service = await this.getOrCreateChatProvider(provider); if (service) { log.info(`[AIServiceManager] Trying provider ${provider} with options.stream: ${options.stream}`); return await service.generateChatCompletion(messages, options); @@ -383,7 +394,7 @@ export class AIServiceManager implements IAIServiceManager { } /** - * Get or create a chat provider on-demand + * Get or create a chat provider on-demand with inline validation */ private async getOrCreateChatProvider(providerName: ServiceProviders): Promise { // Return existing provider if already created @@ -391,38 +402,54 @@ export class AIServiceManager implements IAIServiceManager { return this.services[providerName]; } - // Create provider on-demand based on configuration + // Create and validate provider on-demand try { + let service: AIService | null = null; + switch (providerName) { - case 'openai': - const openaiApiKey = await options.getOption('openaiApiKey'); - if (openaiApiKey) { - this.services.openai = new OpenAIService(); - log.info('Created OpenAI chat provider on-demand'); - return this.services.openai; + case 'openai': { + const apiKey = await options.getOption('openaiApiKey'); + const baseUrl = await options.getOption('openaiBaseUrl'); + if (!apiKey && !baseUrl) return null; + + service = new OpenAIService(); + // Validate by checking if it's available + if (!service.isAvailable()) { + throw new Error('OpenAI service not available'); } break; + } - case 'anthropic': - const anthropicApiKey = await options.getOption('anthropicApiKey'); - if (anthropicApiKey) { - this.services.anthropic = new AnthropicService(); - log.info('Created Anthropic chat provider on-demand'); - return this.services.anthropic; + case 'anthropic': { + const apiKey = await options.getOption('anthropicApiKey'); + if (!apiKey) return null; + + service = new AnthropicService(); + if (!service.isAvailable()) { + throw new Error('Anthropic service not available'); } break; + } - case 'ollama': - const ollamaBaseUrl = await options.getOption('ollamaBaseUrl'); - if (ollamaBaseUrl) { - this.services.ollama = new OllamaService(); - log.info('Created Ollama chat provider on-demand'); - return this.services.ollama; + case 'ollama': { + const baseUrl = await options.getOption('ollamaBaseUrl'); + if (!baseUrl) return null; + + service = new OllamaService(); + if (!service.isAvailable()) { + throw new Error('Ollama service not available'); } break; + } + } + + if (service) { + this.services[providerName] = service; + log.info(`Created and validated ${providerName} chat provider`); + return service; } } catch (error: any) { - log.error(`Error creating ${providerName} provider on-demand: ${error.message || 'Unknown error'}`); + log.error(`Failed to create ${providerName} chat provider: ${error.message || 'Unknown error'}`); } return null; diff --git a/apps/server/src/services/llm/index_service.ts b/apps/server/src/services/llm/index_service.ts index 7786a2cc7..6887fbc36 100644 --- a/apps/server/src/services/llm/index_service.ts +++ b/apps/server/src/services/llm/index_service.ts @@ -48,53 +48,16 @@ export class IndexService { async initialize() { if (this.initialized) return; - try { - // Check if database is initialized before proceeding - if (!sqlInit.isDbInitialized()) { - log.info("Index service: Database not initialized yet, skipping initialization"); - return; - } + // Setup event listeners for note changes + this.setupEventListeners(); - const aiEnabled = options.getOptionOrNull('aiEnabled') === "true"; - if (!aiEnabled) { - log.info("Index service: AI features disabled, skipping initialization"); - return; - } - - // Check if embedding system is ready - if (!(await hasWorkingEmbeddingProviders())) { - log.info("Index service: No working embedding providers available, skipping initialization"); - return; - } - - const providers = await providerManager.getEnabledEmbeddingProviders(); - if (!providers || providers.length === 0) { - log.info("Index service: No enabled embedding providers, skipping initialization"); - return; - } - - // Check if this instance should process embeddings - const embeddingLocation = await options.getOption('embeddingGenerationLocation') || 'client'; - const isSyncServer = await this.isSyncServerForEmbeddings(); - const shouldProcessEmbeddings = embeddingLocation === 'client' || isSyncServer; - - // Setup automatic indexing if enabled and this instance should process embeddings - if (await options.getOptionBool('embeddingAutoUpdateEnabled') && shouldProcessEmbeddings) { - this.setupAutomaticIndexing(); - log.info(`Index service: Automatic indexing enabled, processing embeddings ${isSyncServer ? 'as sync server' : 'as client'}`); - } else if (await options.getOptionBool('embeddingAutoUpdateEnabled')) { - log.info("Index service: Automatic indexing enabled, but this instance is not configured to process embeddings"); - } - - // Listen for note changes to update index - this.setupEventListeners(); - - this.initialized = true; - log.info("Index service initialized successfully"); - } catch (error: any) { - log.error(`Error initializing index service: ${error.message || "Unknown error"}`); - throw error; + // Setup automatic indexing if enabled + if (await options.getOptionBool('embeddingAutoUpdateEnabled')) { + this.setupAutomaticIndexing(); } + + this.initialized = true; + log.info("Index service initialized"); } /** @@ -147,23 +110,7 @@ export class IndexService { this.automaticIndexingInterval = setInterval(async () => { try { if (!this.indexingInProgress) { - // Check if this instance should process embeddings - const embeddingLocation = await options.getOption('embeddingGenerationLocation') || 'client'; - const isSyncServer = await this.isSyncServerForEmbeddings(); - const shouldProcessEmbeddings = embeddingLocation === 'client' || isSyncServer; - - if (!shouldProcessEmbeddings) { - // This instance is not configured to process embeddings - return; - } - - const stats = await vectorStore.getEmbeddingStats(); - - // Only run automatic indexing if we're below 95% completion - if (stats.percentComplete < 95) { - log.info(`Starting automatic indexing (current completion: ${stats.percentComplete}%)`); - await this.runBatchIndexing(50); // Process 50 notes at a time - } + await this.runBatchIndexing(50); // Processing logic handles sync server checks } } catch (error: any) { log.error(`Error in automatic indexing: ${error.message || "Unknown error"}`); @@ -498,35 +445,14 @@ export class IndexService { } try { - // Get all enabled embedding providers - const providers = await providerManager.getEnabledEmbeddingProviders(); - if (!providers || providers.length === 0) { - throw new Error("No embedding providers available"); - } - - // Get the selected embedding provider - const options = (await import('../options.js')).default; + // Get the selected embedding provider on-demand const selectedEmbeddingProvider = await options.getOption('embeddingSelectedProvider'); - let provider; - - if (selectedEmbeddingProvider) { - // Try to use the selected provider - const enabledProviders = await providerManager.getEnabledEmbeddingProviders(); - provider = enabledProviders.find(p => p.name === selectedEmbeddingProvider); - - if (!provider) { - log.info(`Selected embedding provider ${selectedEmbeddingProvider} is not available, using first enabled provider`); - // Fall back to first enabled provider - provider = providers[0]; - } - } else { - // No provider selected, use first available provider - log.info('No embedding provider selected, using first available provider'); - provider = providers[0]; - } + const provider = selectedEmbeddingProvider + ? await providerManager.getOrCreateEmbeddingProvider(selectedEmbeddingProvider) + : (await providerManager.getEnabledEmbeddingProviders())[0]; if (!provider) { - throw new Error("No suitable embedding provider found"); + throw new Error("No embedding provider available"); } log.info(`Searching with embedding provider: ${provider.name}, model: ${provider.getConfig().model}`); @@ -684,6 +610,12 @@ export class IndexService { } try { + // Get embedding providers on-demand + const providers = await providerManager.getEnabledEmbeddingProviders(); + if (providers.length === 0) { + return "I don't have access to your note embeddings. Please configure an embedding provider in your AI settings."; + } + // Find similar notes to the query const similarNotes = await this.findSimilarNotes( query, @@ -819,9 +751,13 @@ export class IndexService { // Get complete note context for indexing const context = await vectorStore.getNoteEmbeddingContext(noteId); - // Queue note for embedding with all available providers - const providers = await providerManager.getEnabledEmbeddingProviders(); - for (const provider of providers) { + // Generate embedding with the selected provider + const selectedEmbeddingProvider = await options.getOption('embeddingSelectedProvider'); + const provider = selectedEmbeddingProvider + ? await providerManager.getOrCreateEmbeddingProvider(selectedEmbeddingProvider) + : (await providerManager.getEnabledEmbeddingProviders())[0]; + + if (provider) { try { const embedding = await provider.generateNoteEmbeddings(context); if (embedding) { @@ -851,7 +787,7 @@ export class IndexService { async startEmbeddingGeneration() { try { log.info("Starting embedding generation system"); - + const aiEnabled = options.getOptionOrNull('aiEnabled') === "true"; if (!aiEnabled) { log.error("Cannot start embedding generation - AI features are disabled"); @@ -873,16 +809,13 @@ export class IndexService { return; } - // Verify providers are available - if (!(await hasWorkingEmbeddingProviders())) { - throw new Error("No working embedding providers available"); - } - + // Get embedding providers (will be created on-demand when needed) const providers = await providerManager.getEnabledEmbeddingProviders(); if (providers.length === 0) { - throw new Error("No embedding providers available"); + log.info("No embedding providers configured, but continuing initialization"); + } else { + log.info(`Found ${providers.length} embedding providers: ${providers.map(p => p.name).join(', ')}`); } - log.info(`Found ${providers.length} embedding providers: ${providers.map(p => p.name).join(', ')}`); // Setup automatic indexing if enabled if (await options.getOptionBool('embeddingAutoUpdateEnabled')) { @@ -899,10 +832,10 @@ export class IndexService { // Queue notes that don't have embeddings for current providers await this.queueNotesForMissingEmbeddings(); - + // Start processing the queue immediately await this.runBatchIndexing(20); - + log.info("Embedding generation started successfully"); } catch (error: any) { log.error(`Error starting embedding generation: ${error.message || "Unknown error"}`); @@ -919,24 +852,24 @@ export class IndexService { try { // Wait for becca to be fully loaded before accessing notes await beccaLoader.beccaLoaded; - + // Get all non-deleted notes const allNotes = Object.values(becca.notes).filter(note => !note.isDeleted); - + // Get enabled providers const providers = await providerManager.getEnabledEmbeddingProviders(); if (providers.length === 0) { return; } - + let queuedCount = 0; let excludedCount = 0; - + // Process notes in batches to avoid overwhelming the system const batchSize = 100; for (let i = 0; i < allNotes.length; i += batchSize) { const batch = allNotes.slice(i, i + batchSize); - + for (const note of batch) { try { // Skip notes excluded from AI @@ -944,10 +877,10 @@ export class IndexService { excludedCount++; continue; } - + // Check if note needs embeddings for any enabled provider let needsEmbedding = false; - + for (const provider of providers) { const config = provider.getConfig(); const existingEmbedding = await vectorStore.getEmbeddingForNote( @@ -955,13 +888,13 @@ export class IndexService { provider.name, config.model ); - + if (!existingEmbedding) { needsEmbedding = true; break; } } - + if (needsEmbedding) { await vectorStore.queueNoteForEmbedding(note.noteId, 'UPDATE'); queuedCount++; @@ -970,7 +903,7 @@ export class IndexService { log.error(`Error checking embeddings for note ${note.noteId}: ${error.message || 'Unknown error'}`); } } - + } } catch (error: any) { log.error(`Error queuing notes for missing embeddings: ${error.message || 'Unknown error'}`); @@ -1005,7 +938,7 @@ export class IndexService { async stopEmbeddingGeneration() { try { log.info("Stopping embedding generation system"); - + // Clear automatic indexing interval if (this.automaticIndexingInterval) { clearInterval(this.automaticIndexingInterval); @@ -1023,7 +956,7 @@ export class IndexService { // Mark as not indexing this.indexingInProgress = false; this.indexRebuildInProgress = false; - + log.info("Embedding generation stopped successfully"); } catch (error: any) { log.error(`Error stopping embedding generation: ${error.message || "Unknown error"}`); diff --git a/apps/server/src/services/llm/provider_validation.ts b/apps/server/src/services/llm/provider_validation.ts index 1cab7bb21..6a94153e8 100644 --- a/apps/server/src/services/llm/provider_validation.ts +++ b/apps/server/src/services/llm/provider_validation.ts @@ -18,7 +18,7 @@ export interface ProviderValidationResult { } /** - * Validate all available providers without throwing errors + * Simplified provider validation - just checks configuration without creating providers */ export async function validateProviders(): Promise { const result: ProviderValidationResult = { @@ -37,14 +37,12 @@ export async function validateProviders(): Promise { return result; } - // Validate embedding providers - await validateEmbeddingProviders(result); - - // Validate chat providers - await validateChatProviders(result); + // Check configuration only - don't create providers + await checkEmbeddingProviderConfigs(result); + await checkChatProviderConfigs(result); - // Determine if we have any valid providers - result.hasValidProviders = result.validEmbeddingProviders.length > 0 || result.validChatProviders.length > 0; + // Determine if we have any valid providers based on configuration + result.hasValidProviders = result.validChatProviders.length > 0; if (!result.hasValidProviders) { result.errors.push("No valid AI providers are configured"); @@ -58,241 +56,80 @@ export async function validateProviders(): Promise { } /** - * Validate embedding providers + * Check embedding provider configurations without creating providers */ -async function validateEmbeddingProviders(result: ProviderValidationResult): Promise { +async function checkEmbeddingProviderConfigs(result: ProviderValidationResult): Promise { try { - // Import provider classes and check configurations - const { OpenAIEmbeddingProvider } = await import("./embeddings/providers/openai.js"); - const { OllamaEmbeddingProvider } = await import("./embeddings/providers/ollama.js"); - const { VoyageEmbeddingProvider } = await import("./embeddings/providers/voyage.js"); + // Check OpenAI embedding configuration + const openaiApiKey = await options.getOption('openaiApiKey'); + const openaiBaseUrl = await options.getOption('openaiBaseUrl'); + if (openaiApiKey || openaiBaseUrl) { + if (!openaiApiKey) { + result.warnings.push("OpenAI embedding: No API key (may work with compatible endpoints)"); + } + log.info("OpenAI embedding provider configuration available"); + } - // Check OpenAI embedding provider - await validateOpenAIEmbeddingProvider(result, OpenAIEmbeddingProvider); - - // Check Ollama embedding provider - await validateOllamaEmbeddingProvider(result, OllamaEmbeddingProvider); - - // Check Voyage embedding provider - await validateVoyageEmbeddingProvider(result, VoyageEmbeddingProvider); + // Check Ollama embedding configuration + const ollamaEmbeddingBaseUrl = await options.getOption('ollamaEmbeddingBaseUrl'); + if (ollamaEmbeddingBaseUrl) { + log.info("Ollama embedding provider configuration available"); + } - // Local provider is always available as fallback - await validateLocalEmbeddingProvider(result); + // Check Voyage embedding configuration + const voyageApiKey = await options.getOption('voyageApiKey' as any); + if (voyageApiKey) { + log.info("Voyage embedding provider configuration available"); + } + + // Local provider is always available + log.info("Local embedding provider available as fallback"); } catch (error: any) { - result.errors.push(`Error validating embedding providers: ${error.message || 'Unknown error'}`); + result.errors.push(`Error checking embedding provider configs: ${error.message || 'Unknown error'}`); } } /** - * Validate chat providers + * Check chat provider configurations without creating providers */ -async function validateChatProviders(result: ProviderValidationResult): Promise { +async function checkChatProviderConfigs(result: ProviderValidationResult): Promise { try { // Check OpenAI chat provider const openaiApiKey = await options.getOption('openaiApiKey'); const openaiBaseUrl = await options.getOption('openaiBaseUrl'); if (openaiApiKey || openaiBaseUrl) { - if (!openaiApiKey && !openaiBaseUrl) { - result.warnings.push("OpenAI chat provider: No API key or base URL configured"); - } else if (!openaiApiKey) { - result.warnings.push("OpenAI chat provider: No API key configured (may work with compatible endpoints)"); - result.validChatProviders.push('openai'); - } else { - result.validChatProviders.push('openai'); + if (!openaiApiKey) { + result.warnings.push("OpenAI chat: No API key (may work with compatible endpoints)"); } + result.validChatProviders.push('openai'); } // Check Anthropic chat provider const anthropicApiKey = await options.getOption('anthropicApiKey'); if (anthropicApiKey) { result.validChatProviders.push('anthropic'); - } else { - result.warnings.push("Anthropic chat provider: No API key configured"); } // Check Ollama chat provider const ollamaBaseUrl = await options.getOption('ollamaBaseUrl'); if (ollamaBaseUrl) { result.validChatProviders.push('ollama'); - } else { - result.warnings.push("Ollama chat provider: No base URL configured"); + } + + if (result.validChatProviders.length === 0) { + result.warnings.push("No chat providers configured. Please configure at least one provider."); } } catch (error: any) { - result.errors.push(`Error validating chat providers: ${error.message || 'Unknown error'}`); + result.errors.push(`Error checking chat provider configs: ${error.message || 'Unknown error'}`); } } -/** - * Validate OpenAI embedding provider - */ -async function validateOpenAIEmbeddingProvider( - result: ProviderValidationResult, - OpenAIEmbeddingProvider: any -): Promise { - try { - const openaiApiKey = await options.getOption('openaiApiKey'); - const openaiBaseUrl = await options.getOption('openaiBaseUrl'); - - if (openaiApiKey || openaiBaseUrl) { - const openaiModel = await options.getOption('openaiEmbeddingModel'); - const finalBaseUrl = openaiBaseUrl || 'https://api.openai.com/v1'; - - if (!openaiApiKey) { - result.warnings.push("OpenAI embedding provider: No API key configured (may work with compatible endpoints)"); - } - - const provider = new OpenAIEmbeddingProvider({ - model: openaiModel, - dimension: 1536, - type: 'float32', - apiKey: openaiApiKey || '', - baseUrl: finalBaseUrl - }); - - result.validEmbeddingProviders.push(provider); - log.info(`Validated OpenAI embedding provider: ${openaiModel} at ${finalBaseUrl}`); - } else { - result.warnings.push("OpenAI embedding provider: No API key or base URL configured"); - } - } catch (error: any) { - result.errors.push(`OpenAI embedding provider validation failed: ${error.message || 'Unknown error'}`); - } -} /** - * Validate Ollama embedding provider - */ -async function validateOllamaEmbeddingProvider( - result: ProviderValidationResult, - OllamaEmbeddingProvider: any -): Promise { - try { - const ollamaEmbeddingBaseUrl = await options.getOption('ollamaEmbeddingBaseUrl'); - - if (ollamaEmbeddingBaseUrl) { - const embeddingModel = await options.getOption('ollamaEmbeddingModel'); - - try { - const provider = new OllamaEmbeddingProvider({ - model: embeddingModel, - dimension: 768, - type: 'float32', - baseUrl: ollamaEmbeddingBaseUrl - }); - - // Try to initialize to validate connection - await provider.initialize(); - result.validEmbeddingProviders.push(provider); - log.info(`Validated Ollama embedding provider: ${embeddingModel} at ${ollamaEmbeddingBaseUrl}`); - } catch (error: any) { - result.warnings.push(`Ollama embedding provider initialization failed: ${error.message || 'Unknown error'}`); - } - } else { - result.warnings.push("Ollama embedding provider: No base URL configured"); - } - } catch (error: any) { - result.errors.push(`Ollama embedding provider validation failed: ${error.message || 'Unknown error'}`); - } -} - -/** - * Validate Voyage embedding provider - */ -async function validateVoyageEmbeddingProvider( - result: ProviderValidationResult, - VoyageEmbeddingProvider: any -): Promise { - try { - const voyageApiKey = await options.getOption('voyageApiKey' as any); - - if (voyageApiKey) { - const voyageModel = await options.getOption('voyageEmbeddingModel') || 'voyage-2'; - - const provider = new VoyageEmbeddingProvider({ - model: voyageModel, - dimension: 1024, - type: 'float32', - apiKey: voyageApiKey, - baseUrl: 'https://api.voyageai.com/v1' - }); - - result.validEmbeddingProviders.push(provider); - log.info(`Validated Voyage embedding provider: ${voyageModel}`); - } else { - result.warnings.push("Voyage embedding provider: No API key configured"); - } - } catch (error: any) { - result.errors.push(`Voyage embedding provider validation failed: ${error.message || 'Unknown error'}`); - } -} - -/** - * Validate local embedding provider (always available as fallback) - */ -async function validateLocalEmbeddingProvider(result: ProviderValidationResult): Promise { - try { - // Simple local embedding provider implementation - class SimpleLocalEmbeddingProvider { - name = "local"; - config = { - model: 'local', - dimension: 384, - type: 'float32' as const - }; - - getConfig() { - return this.config; - } - - getNormalizationStatus() { - return 0; // NormalizationStatus.NEVER - } - - async generateEmbeddings(text: string): Promise { - const result = new Float32Array(this.config.dimension); - for (let i = 0; i < result.length; i++) { - const charSum = Array.from(text).reduce((sum, char, idx) => - sum + char.charCodeAt(0) * Math.sin(idx * 0.1), 0); - result[i] = Math.sin(i * 0.1 + charSum * 0.01); - } - return result; - } - - async generateBatchEmbeddings(texts: string[]): Promise { - return Promise.all(texts.map(text => this.generateEmbeddings(text))); - } - - async generateNoteEmbeddings(context: any): Promise { - const text = (context.title || "") + " " + (context.content || ""); - return this.generateEmbeddings(text); - } - - async generateBatchNoteEmbeddings(contexts: any[]): Promise { - return Promise.all(contexts.map(context => this.generateNoteEmbeddings(context))); - } - } - - const localProvider = new SimpleLocalEmbeddingProvider(); - result.validEmbeddingProviders.push(localProvider as any); - log.info("Validated local embedding provider as fallback"); - } catch (error: any) { - result.errors.push(`Local embedding provider validation failed: ${error.message || 'Unknown error'}`); - } -} - -/** - * Check if any working providers are available for embeddings - */ -export async function hasWorkingEmbeddingProviders(): Promise { - const validation = await validateProviders(); - return validation.validEmbeddingProviders.length > 0; -} - -/** - * Check if any working providers are available for chat + * Check if any chat providers are configured */ export async function hasWorkingChatProviders(): Promise { const validation = await validateProviders(); @@ -300,11 +137,21 @@ export async function hasWorkingChatProviders(): Promise { } /** - * Get only the working embedding providers + * Check if any embedding providers are configured (simplified) */ -export async function getWorkingEmbeddingProviders(): Promise { - const validation = await validateProviders(); - return validation.validEmbeddingProviders; +export async function hasWorkingEmbeddingProviders(): Promise { + if (!(await options.getOptionBool('aiEnabled'))) { + return false; + } + + // Check if any embedding provider is configured + const openaiKey = await options.getOption('openaiApiKey'); + const openaiBaseUrl = await options.getOption('openaiBaseUrl'); + const ollamaUrl = await options.getOption('ollamaEmbeddingBaseUrl'); + const voyageKey = await options.getOption('voyageApiKey' as any); + + // Local provider is always available as fallback + return !!(openaiKey || openaiBaseUrl || ollamaUrl || voyageKey) || true; } /** diff --git a/apps/server/src/services/llm/providers/providers.ts b/apps/server/src/services/llm/providers/providers.ts index fd7c603fd..dae8b34a0 100644 --- a/apps/server/src/services/llm/providers/providers.ts +++ b/apps/server/src/services/llm/providers/providers.ts @@ -124,118 +124,129 @@ export function getEmbeddingProvider(name: string): EmbeddingProvider | undefine } /** - * Create providers on-demand based on current options + * Get or create a specific embedding provider with inline validation */ -export async function createProvidersFromCurrentOptions(): Promise { - const result: EmbeddingProvider[] = []; - - try { - // Create Ollama provider if embedding base URL is configured - const ollamaEmbeddingBaseUrl = await options.getOption('ollamaEmbeddingBaseUrl'); - if (ollamaEmbeddingBaseUrl) { - const embeddingModel = await options.getOption('ollamaEmbeddingModel'); - - try { - const ollamaProvider = new OllamaEmbeddingProvider({ - model: embeddingModel, - dimension: 768, // Initial value, will be updated during initialization - type: 'float32', - baseUrl: ollamaEmbeddingBaseUrl - }); - - await ollamaProvider.initialize(); - registerEmbeddingProvider(ollamaProvider); - result.push(ollamaProvider); - log.info(`Created Ollama provider on-demand: ${embeddingModel} at ${ollamaEmbeddingBaseUrl}`); - } catch (error: any) { - log.error(`Error creating Ollama embedding provider on-demand: ${error.message || 'Unknown error'}`); - } - } - - // Create OpenAI provider even without API key (for OpenAI-compatible endpoints) - const openaiApiKey = await options.getOption('openaiApiKey'); - const openaiBaseUrl = await options.getOption('openaiBaseUrl'); - - // Only create OpenAI provider if base URL is set or API key is provided - if (openaiApiKey || openaiBaseUrl) { - const openaiModel = await options.getOption('openaiEmbeddingModel') - const finalBaseUrl = openaiBaseUrl || 'https://api.openai.com/v1'; - - if (!openaiApiKey) { - log.info('Creating OpenAI embedding provider without API key. This may cause issues with official OpenAI endpoints.'); - } - - const openaiProvider = new OpenAIEmbeddingProvider({ - model: openaiModel, - dimension: 1536, - type: 'float32', - apiKey: openaiApiKey || '', // Default to empty string - baseUrl: finalBaseUrl - }); - - registerEmbeddingProvider(openaiProvider); - result.push(openaiProvider); - log.info(`Created OpenAI provider on-demand: ${openaiModel} at ${finalBaseUrl}`); - } - - // Create Voyage provider if API key is configured - const voyageApiKey = await options.getOption('voyageApiKey' as any); - if (voyageApiKey) { - const voyageModel = await options.getOption('voyageEmbeddingModel') || 'voyage-2'; - const voyageBaseUrl = 'https://api.voyageai.com/v1'; - - const voyageProvider = new VoyageEmbeddingProvider({ - model: voyageModel, - dimension: 1024, - type: 'float32', - apiKey: voyageApiKey, - baseUrl: voyageBaseUrl - }); - - registerEmbeddingProvider(voyageProvider); - result.push(voyageProvider); - log.info(`Created Voyage provider on-demand: ${voyageModel}`); - } - - // Always include local provider as fallback - if (!providers.has('local')) { - const localProvider = new SimpleLocalEmbeddingProvider({ - model: 'local', - dimension: 384, - type: 'float32' - }); - registerEmbeddingProvider(localProvider); - result.push(localProvider); - log.info(`Created local provider on-demand as fallback`); - } else { - result.push(providers.get('local')!); - } - - } catch (error: any) { - log.error(`Error creating providers from current options: ${error.message || 'Unknown error'}`); +export async function getOrCreateEmbeddingProvider(providerName: string): Promise { + // Return existing provider if already created and valid + const existing = providers.get(providerName); + if (existing) { + return existing; } - return result; + // Create and validate provider on-demand + try { + let provider: EmbeddingProvider | null = null; + + switch (providerName) { + case 'ollama': { + const baseUrl = await options.getOption('ollamaEmbeddingBaseUrl'); + if (!baseUrl) return null; + + const model = await options.getOption('ollamaEmbeddingModel'); + provider = new OllamaEmbeddingProvider({ + model, + dimension: 768, + type: 'float32', + baseUrl + }); + + // Validate by initializing (if provider supports it) + if ('initialize' in provider && typeof provider.initialize === 'function') { + await provider.initialize(); + } + break; + } + + case 'openai': { + const apiKey = await options.getOption('openaiApiKey'); + const baseUrl = await options.getOption('openaiBaseUrl'); + if (!apiKey && !baseUrl) return null; + + const model = await options.getOption('openaiEmbeddingModel'); + provider = new OpenAIEmbeddingProvider({ + model, + dimension: 1536, + type: 'float32', + apiKey: apiKey || '', + baseUrl: baseUrl || 'https://api.openai.com/v1' + }); + + if (!apiKey) { + log.info('OpenAI embedding provider created without API key for compatible endpoints'); + } + break; + } + + case 'voyage': { + const apiKey = await options.getOption('voyageApiKey' as any); + if (!apiKey) return null; + + const model = await options.getOption('voyageEmbeddingModel') || 'voyage-2'; + provider = new VoyageEmbeddingProvider({ + model, + dimension: 1024, + type: 'float32', + apiKey, + baseUrl: 'https://api.voyageai.com/v1' + }); + break; + } + + case 'local': { + provider = new SimpleLocalEmbeddingProvider({ + model: 'local', + dimension: 384, + type: 'float32' + }); + break; + } + + default: + return null; + } + + if (provider) { + registerEmbeddingProvider(provider); + log.info(`Created and validated ${providerName} embedding provider`); + return provider; + } + } catch (error: any) { + log.error(`Failed to create ${providerName} embedding provider: ${error.message || 'Unknown error'}`); + } + + return null; } /** - * Get all enabled embedding providers + * Get all enabled embedding providers for the specified feature */ -export async function getEnabledEmbeddingProviders(): Promise { +export async function getEnabledEmbeddingProviders(feature: 'embeddings' | 'chat' = 'embeddings'): Promise { if (!(await options.getOptionBool('aiEnabled'))) { return []; } - // First try to get existing registered providers - const existingProviders = Array.from(providers.values()); + const result: EmbeddingProvider[] = []; - // If no providers are registered, create them on-demand from current options - if (existingProviders.length === 0) { - log.info('No providers registered, creating from current options'); - return await createProvidersFromCurrentOptions(); + // Get the selected provider for the feature + const selectedProvider = feature === 'embeddings' + ? await options.getOption('embeddingSelectedProvider') + : await options.getOption('aiSelectedProvider'); + + // Try to get or create the specific selected provider + const provider = await getOrCreateEmbeddingProvider(selectedProvider); + if (!provider) { + throw new Error(`Failed to create selected embedding provider: ${selectedProvider}. Please check your configuration.`); + } + result.push(provider); + + + // Always ensure local provider as fallback + const localProvider = await getOrCreateEmbeddingProvider('local'); + if (localProvider && !result.some(p => p.name === 'local')) { + result.push(localProvider); } - return existingProviders; + return result; } /** @@ -342,7 +353,7 @@ export default { getEmbeddingProviders, getEmbeddingProvider, getEnabledEmbeddingProviders, - createProvidersFromCurrentOptions, + getOrCreateEmbeddingProvider, createEmbeddingProviderConfig, updateEmbeddingProviderConfig, deleteEmbeddingProviderConfig, From 6bc9b3c184ea204b1158c82f1bccc4d0a282cd60 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Sat, 7 Jun 2025 00:02:26 +0000 Subject: [PATCH 15/18] feat(llm): resolve sending double headers in responses, and not being able to send requests to ollama --- .../options/ai_settings/ai_settings_widget.ts | 2 +- .../options/ai_settings/providers.ts | 61 ++++-- apps/server/src/routes/api/llm.ts | 90 +++----- .../src/services/llm/ai_service_manager.ts | 192 ++++++++---------- .../services/llm/chat/rest_chat_service.ts | 29 +-- .../llm/config/configuration_manager.ts | 36 ++-- 6 files changed, 179 insertions(+), 231 deletions(-) diff --git a/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts b/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts index 4f63bfcb1..ec6e3e576 100644 --- a/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts +++ b/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts @@ -376,7 +376,7 @@ export default class AiSettingsWidget extends OptionsWidget { embeddingWarnings.push(t("ai_llm.empty_key_warning.voyage")); } - if (selectedEmbeddingProvider === 'ollama' && !this.$widget.find('.ollama-base-url').val()) { + if (selectedEmbeddingProvider === 'ollama' && !this.$widget.find('.ollama-embedding-base-url').val()) { embeddingWarnings.push(t("ai_llm.empty_key_warning.ollama")); } } diff --git a/apps/client/src/widgets/type_widgets/options/ai_settings/providers.ts b/apps/client/src/widgets/type_widgets/options/ai_settings/providers.ts index c3b35e34d..281569bd6 100644 --- a/apps/client/src/widgets/type_widgets/options/ai_settings/providers.ts +++ b/apps/client/src/widgets/type_widgets/options/ai_settings/providers.ts @@ -240,40 +240,65 @@ export class ProviderService { } try { - const ollamaBaseUrl = this.$widget.find('.ollama-base-url').val() as string; + // Determine which URL to use based on the current context + // If we're in the embedding provider context, use the embedding base URL + // Otherwise, use the general base URL + const selectedAiProvider = this.$widget.find('.ai-selected-provider').val() as string; + const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; + + let ollamaBaseUrl: string; + + // If embedding provider is Ollama and it's visible, use embedding URL + const $embeddingOllamaSettings = this.$widget.find('.ollama-embedding-provider-settings'); + if (selectedEmbeddingProvider === 'ollama' && $embeddingOllamaSettings.is(':visible')) { + ollamaBaseUrl = this.$widget.find('.ollama-embedding-base-url').val() as string; + } else { + ollamaBaseUrl = this.$widget.find('.ollama-base-url').val() as string; + } + const response = await server.get(`llm/providers/ollama/models?baseUrl=${encodeURIComponent(ollamaBaseUrl)}`); if (response && response.success && response.models && response.models.length > 0) { + // Update both embedding model dropdowns const $embedModelSelect = this.$widget.find('.ollama-embedding-model'); + const $chatEmbedModelSelect = this.$widget.find('.ollama-chat-embedding-model'); + const currentValue = $embedModelSelect.val(); + const currentChatEmbedValue = $chatEmbedModelSelect.val(); - // Clear existing options - $embedModelSelect.empty(); - - // Add embedding-specific models first + // Prepare embedding models const embeddingModels = response.models.filter(model => model.name.includes('embed') || model.name.includes('bert')); - - embeddingModels.forEach(model => { - $embedModelSelect.append(``); - }); - - if (embeddingModels.length > 0) { - // Add separator if we have embedding models - $embedModelSelect.append(``); - } - - // Then add general models which can be used for embeddings too + const generalModels = response.models.filter(model => !model.name.includes('embed') && !model.name.includes('bert')); + // Update .ollama-embedding-model dropdown (embedding provider settings) + $embedModelSelect.empty(); + embeddingModels.forEach(model => { + $embedModelSelect.append(``); + }); + if (embeddingModels.length > 0) { + $embedModelSelect.append(``); + } generalModels.forEach(model => { $embedModelSelect.append(``); }); - - // Try to restore the previously selected value this.ensureSelectedValue($embedModelSelect, currentValue, 'ollamaEmbeddingModel'); + // Update .ollama-chat-embedding-model dropdown (general Ollama provider settings) + $chatEmbedModelSelect.empty(); + embeddingModels.forEach(model => { + $chatEmbedModelSelect.append(``); + }); + if (embeddingModels.length > 0) { + $chatEmbedModelSelect.append(``); + } + generalModels.forEach(model => { + $chatEmbedModelSelect.append(``); + }); + this.ensureSelectedValue($chatEmbedModelSelect, currentChatEmbedValue, 'ollamaEmbeddingModel'); + // Also update the LLM model dropdown const $modelSelect = this.$widget.find('.ollama-default-model'); const currentModelValue = $modelSelect.val(); diff --git a/apps/server/src/routes/api/llm.ts b/apps/server/src/routes/api/llm.ts index c21426a66..f586b85d6 100644 --- a/apps/server/src/routes/api/llm.ts +++ b/apps/server/src/routes/api/llm.ts @@ -825,7 +825,10 @@ async function streamMessage(req: Request, res: Response) { success: true, message: 'Streaming initiated successfully' }); - log.info(`Sent immediate success response for streaming setup`); + + // Mark response as handled to prevent apiResultHandler from processing it again + (res as any).triliumResponseHandled = true; + // Create a new response object for streaming through WebSocket only // We won't use HTTP streaming since we've already sent the HTTP response @@ -889,78 +892,33 @@ async function streamMessage(req: Request, res: Response) { thinking: showThinking ? 'Initializing streaming LLM response...' : undefined }); - // Instead of trying to reimplement the streaming logic ourselves, - // delegate to restChatService but set up the correct protocol: - // 1. We've already sent a success response to the initial POST - // 2. Now we'll have restChatService process the actual streaming through WebSocket + // Process the LLM request using the existing service but with streaming setup + // Since we've already sent the initial HTTP response, we'll use the WebSocket for streaming try { - // Import the WebSocket service for sending messages - const wsService = (await import('../../services/ws.js')).default; - - // Create a simple pass-through response object that won't write to the HTTP response - // but will allow restChatService to send WebSocket messages - const dummyResponse = { - writableEnded: false, - // Implement methods that would normally be used by restChatService - write: (_chunk: string) => { - // Silent no-op - we're only using WebSocket - return true; + // Call restChatService with streaming mode enabled + // The important part is setting method to GET to indicate streaming mode + await restChatService.handleSendMessage({ + ...req, + method: 'GET', // Indicate streaming mode + query: { + ...req.query, + stream: 'true' // Add the required stream parameter }, - end: (_chunk?: string) => { - // Log when streaming is complete via WebSocket - log.info(`[${chatNoteId}] Completed HTTP response handling during WebSocket streaming`); - return dummyResponse; + body: { + content: enhancedContent, + useAdvancedContext: useAdvancedContext === true, + showThinking: showThinking === true }, - setHeader: (name: string, _value: string) => { - // Only log for content-type to reduce noise - if (name.toLowerCase() === 'content-type') { - log.info(`[${chatNoteId}] Setting up streaming for WebSocket only`); - } - return dummyResponse; - } - }; + params: { chatNoteId } + } as unknown as Request, res); + } catch (streamError) { + log.error(`Error during WebSocket streaming: ${streamError}`); - // Process the streaming now through WebSocket only - try { - log.info(`[${chatNoteId}] Processing LLM streaming through WebSocket after successful initiation at ${new Date().toISOString()}`); - - // Call restChatService with our enhanced request and dummy response - // The important part is setting method to GET to indicate streaming mode - await restChatService.handleSendMessage({ - ...req, - method: 'GET', // Indicate streaming mode - query: { - ...req.query, - stream: 'true' // Add the required stream parameter - }, - body: { - content: enhancedContent, - useAdvancedContext: useAdvancedContext === true, - showThinking: showThinking === true - }, - params: { chatNoteId } - } as unknown as Request, dummyResponse as unknown as Response); - - log.info(`[${chatNoteId}] WebSocket streaming completed at ${new Date().toISOString()}`); - } catch (streamError) { - log.error(`[${chatNoteId}] Error during WebSocket streaming: ${streamError}`); - - // Send error message through WebSocket - wsService.sendMessageToAllClients({ - type: 'llm-stream', - chatNoteId: chatNoteId, - error: `Error during streaming: ${streamError}`, - done: true - }); - } - } catch (error) { - log.error(`Error during streaming: ${error}`); - - // Send error to client via WebSocket + // Send error message through WebSocket wsService.sendMessageToAllClients({ type: 'llm-stream', chatNoteId: chatNoteId, - error: `Error processing message: ${error}`, + error: `Error during streaming: ${streamError}`, done: true }); } diff --git a/apps/server/src/services/llm/ai_service_manager.ts b/apps/server/src/services/llm/ai_service_manager.ts index 222f91dfd..394523d8f 100644 --- a/apps/server/src/services/llm/ai_service_manager.ts +++ b/apps/server/src/services/llm/ai_service_manager.ts @@ -45,13 +45,9 @@ interface NoteContext { export class AIServiceManager implements IAIServiceManager { private services: Partial> = {}; - private providerOrder: ServiceProviders[] = []; // Will be populated from configuration private initialized = false; constructor() { - // Initialize provider order immediately - this.updateProviderOrder(); - // Initialize tools immediately this.initializeTools().catch(error => { log.error(`Error initializing LLM tools during AIServiceManager construction: ${error.message || String(error)}`); @@ -59,6 +55,8 @@ export class AIServiceManager implements IAIServiceManager { // Set up event listener for provider changes this.setupProviderChangeListener(); + + this.initialized = true; } /** @@ -83,44 +81,18 @@ export class AIServiceManager implements IAIServiceManager { } /** - * Update the provider order using the new configuration system (single provider) + * Get the currently selected provider using the new configuration system */ - async updateProviderOrderAsync(): Promise { + async getSelectedProviderAsync(): Promise { try { const selectedProvider = await getSelectedProvider(); - if (selectedProvider) { - this.providerOrder = [selectedProvider as ServiceProviders]; - log.info(`Updated provider order: ${selectedProvider}`); - } else { - this.providerOrder = []; - log.info('No provider selected'); - } - this.initialized = true; + return selectedProvider as ServiceProviders || null; } catch (error) { log.error(`Failed to get selected provider: ${error}`); - // Keep empty order, will be handled gracefully by other methods - this.providerOrder = []; - this.initialized = true; + return null; } } - /** - * Update the provider precedence order (legacy sync version) - * Returns true if successful, false if options not available yet - */ - updateProviderOrder(): boolean { - if (this.initialized) { - return true; - } - - // Use async version but don't wait - this.updateProviderOrderAsync().catch(error => { - log.error(`Error in async provider order update: ${error}`); - }); - - return true; - } - /** * Validate AI configuration using the new configuration system */ @@ -162,16 +134,44 @@ export class AIServiceManager implements IAIServiceManager { * Ensure manager is initialized before using */ private ensureInitialized() { - if (!this.initialized) { - this.updateProviderOrder(); + // No longer needed with simplified approach + } + + /** + * Get or create any available AI service following the simplified pattern + * Returns a service or throws a meaningful error + */ + async getOrCreateAnyService(): Promise { + this.ensureInitialized(); + + // Get the selected provider using the new configuration system + const selectedProvider = await this.getSelectedProviderAsync(); + + + if (!selectedProvider) { + throw new Error('No AI provider is selected. Please select a provider (OpenAI, Anthropic, or Ollama) in your AI settings.'); + } + + try { + const service = await this.getOrCreateChatProvider(selectedProvider); + if (service) { + return service; + } + throw new Error(`Failed to create ${selectedProvider} service`); + } catch (error) { + log.error(`Provider ${selectedProvider} not available: ${error}`); + throw new Error(`Selected AI provider (${selectedProvider}) is not available. Please check your configuration: ${error}`); } } /** - * Check if any AI service is available + * Check if any AI service is available (legacy method for backward compatibility) */ isAnyServiceAvailable(): boolean { - return Object.values(this.services).some(service => service.isAvailable()); + this.ensureInitialized(); + + // Check if we have the selected provider available + return this.getAvailableProviders().length > 0; } /** @@ -235,25 +235,27 @@ export class AIServiceManager implements IAIServiceManager { throw new Error('No messages provided for chat completion'); } - // Try providers in order of preference - const availableProviders = this.getAvailableProviders(); - - if (availableProviders.length === 0) { - throw new Error('No AI providers are available. Please check your AI settings.'); + // Get the selected provider + const selectedProvider = await this.getSelectedProviderAsync(); + + if (!selectedProvider) { + throw new Error('No AI provider is selected. Please select a provider in your AI settings.'); + } + + // Check if the selected provider is available + const availableProviders = this.getAvailableProviders(); + if (!availableProviders.includes(selectedProvider)) { + throw new Error(`Selected AI provider (${selectedProvider}) is not available. Please check your configuration.`); } - - // Sort available providers by precedence - const sortedProviders = this.providerOrder - .filter(provider => availableProviders.includes(provider)); // If a specific provider is requested and available, use it if (options.model && options.model.includes(':')) { // Use the new configuration system to parse model identifier const modelIdentifier = parseModelIdentifier(options.model); - if (modelIdentifier.provider && availableProviders.includes(modelIdentifier.provider as ServiceProviders)) { + if (modelIdentifier.provider && modelIdentifier.provider === selectedProvider) { try { - const service = this.services[modelIdentifier.provider as ServiceProviders]; + const service = await this.getOrCreateChatProvider(modelIdentifier.provider as ServiceProviders); if (service) { const modifiedOptions = { ...options, model: modelIdentifier.modelId }; log.info(`[AIServiceManager] Using provider ${modelIdentifier.provider} from model prefix with modifiedOptions.stream: ${modifiedOptions.stream}`); @@ -261,42 +263,26 @@ export class AIServiceManager implements IAIServiceManager { } } catch (error) { log.error(`Error with specified provider ${modelIdentifier.provider}: ${error}`); - // If the specified provider fails, continue with the fallback providers + throw new Error(`Failed to use specified provider ${modelIdentifier.provider}: ${error}`); } + } else if (modelIdentifier.provider && modelIdentifier.provider !== selectedProvider) { + throw new Error(`Model specifies provider '${modelIdentifier.provider}' but selected provider is '${selectedProvider}'. Please select the correct provider or use a model without provider prefix.`); } // If not a provider prefix, treat the entire string as a model name and continue with normal provider selection } - // If user has a specific provider selected, try only that one and fail fast - if (this.providerOrder.length === 1 && sortedProviders.length === 1) { - const selectedProvider = sortedProviders[0]; + // Use the selected provider + try { const service = await this.getOrCreateChatProvider(selectedProvider); if (!service) { throw new Error(`Failed to create selected chat provider: ${selectedProvider}. Please check your configuration.`); } log.info(`[AIServiceManager] Using selected provider ${selectedProvider} with options.stream: ${options.stream}`); return await service.generateChatCompletion(messages, options); + } catch (error) { + log.error(`Error with selected provider ${selectedProvider}: ${error}`); + throw new Error(`Selected AI provider (${selectedProvider}) failed: ${error}`); } - - // If no specific provider selected, try each provider in order until one succeeds - let lastError: Error | null = null; - - for (const provider of sortedProviders) { - try { - const service = await this.getOrCreateChatProvider(provider); - if (service) { - log.info(`[AIServiceManager] Trying provider ${provider} with options.stream: ${options.stream}`); - return await service.generateChatCompletion(messages, options); - } - } catch (error) { - log.error(`Error with provider ${provider}: ${error}`); - lastError = error as Error; - // Continue to the next provider - } - } - - // If we get here, all providers failed - throw new Error(`All AI providers failed: ${lastError?.message || 'Unknown error'}`); } setupEventListeners() { @@ -408,8 +394,8 @@ export class AIServiceManager implements IAIServiceManager { switch (providerName) { case 'openai': { - const apiKey = await options.getOption('openaiApiKey'); - const baseUrl = await options.getOption('openaiBaseUrl'); + const apiKey = options.getOption('openaiApiKey'); + const baseUrl = options.getOption('openaiBaseUrl'); if (!apiKey && !baseUrl) return null; service = new OpenAIService(); @@ -421,7 +407,7 @@ export class AIServiceManager implements IAIServiceManager { } case 'anthropic': { - const apiKey = await options.getOption('anthropicApiKey'); + const apiKey = options.getOption('anthropicApiKey'); if (!apiKey) return null; service = new AnthropicService(); @@ -432,7 +418,7 @@ export class AIServiceManager implements IAIServiceManager { } case 'ollama': { - const baseUrl = await options.getOption('ollamaBaseUrl'); + const baseUrl = options.getOption('ollamaBaseUrl'); if (!baseUrl) return null; service = new OllamaService(); @@ -445,7 +431,6 @@ export class AIServiceManager implements IAIServiceManager { if (service) { this.services[providerName] = service; - log.info(`Created and validated ${providerName} chat provider`); return service; } } catch (error: any) { @@ -470,9 +455,6 @@ export class AIServiceManager implements IAIServiceManager { return; } - // Update provider order from configuration - await this.updateProviderOrderAsync(); - // Initialize index service await this.getIndexService().initialize(); @@ -590,18 +572,22 @@ export class AIServiceManager implements IAIServiceManager { if (service && service.isAvailable()) { return service; } + throw new Error(`Specified provider ${provider} is not available`); } - // Otherwise, try providers in the configured order - for (const providerName of this.providerOrder) { - const service = await this.getOrCreateChatProvider(providerName); - if (service && service.isAvailable()) { - return service; - } + // Otherwise, use the selected provider + const selectedProvider = await this.getSelectedProviderAsync(); + if (!selectedProvider) { + throw new Error('No AI provider is selected. Please select a provider in your AI settings.'); + } + + const service = await this.getOrCreateChatProvider(selectedProvider); + if (service && service.isAvailable()) { + return service; } // If no provider is available, throw a clear error - throw new Error('No AI chat providers are available. Please check your AI settings.'); + throw new Error(`Selected AI provider (${selectedProvider}) is not available. Please check your AI settings.`); } /** @@ -611,14 +597,14 @@ export class AIServiceManager implements IAIServiceManager { try { const selectedProvider = await getSelectedProvider(); if (selectedProvider === null) { - // No provider selected, fallback to first available - log.info('No provider selected, using first available provider'); - return this.providerOrder[0]; + // No provider selected, fallback to default + log.info('No provider selected, using default provider'); + return 'openai'; } return selectedProvider; } catch (error) { log.error(`Error getting preferred provider: ${error}`); - return this.providerOrder[0]; + return 'openai'; } } @@ -628,16 +614,18 @@ export class AIServiceManager implements IAIServiceManager { getSelectedProvider(): string { this.ensureInitialized(); - // Return the first available provider in the order - for (const providerName of this.providerOrder) { - const service = this.services[providerName]; - if (service && service.isAvailable()) { - return providerName; + // Try to get the selected provider synchronously + try { + const selectedProvider = options.getOption('aiSelectedProvider'); + if (selectedProvider) { + return selectedProvider; } + } catch (error) { + log.error(`Error getting selected provider: ${error}`); } - // Return the first provider as fallback - return this.providerOrder[0]; + // Return a default if nothing is selected (for backward compatibility) + return 'openai'; } /** @@ -746,9 +734,6 @@ export class AIServiceManager implements IAIServiceManager { const providerManager = await import('./providers/providers.js'); providerManager.clearAllEmbeddingProviders(); - // Update provider order with new configuration - await this.updateProviderOrderAsync(); - log.info('LLM services recreated successfully'); } catch (error) { log.error(`Error recreating LLM services: ${this.handleError(error)}`); @@ -776,6 +761,9 @@ export default { isAnyServiceAvailable(): boolean { return getInstance().isAnyServiceAvailable(); }, + async getOrCreateAnyService(): Promise { + return getInstance().getOrCreateAnyService(); + }, getAvailableProviders() { return getInstance().getAvailableProviders(); }, diff --git a/apps/server/src/services/llm/chat/rest_chat_service.ts b/apps/server/src/services/llm/chat/rest_chat_service.ts index 4176eaffe..1d5434ac1 100644 --- a/apps/server/src/services/llm/chat/rest_chat_service.ts +++ b/apps/server/src/services/llm/chat/rest_chat_service.ts @@ -5,7 +5,7 @@ import log from "../../log.js"; import type { Request, Response } from "express"; import type { Message, ChatCompletionOptions } from "../ai_interface.js"; -import { AIServiceManager } from "../ai_service_manager.js"; +import aiServiceManager from "../ai_service_manager.js"; import { ChatPipeline } from "../pipeline/chat_pipeline.js"; import type { ChatPipelineInput } from "../pipeline/interfaces.js"; import options from "../../options.js"; @@ -33,25 +33,6 @@ class RestChatService { } } - /** - * Check if AI services are available - */ - safelyUseAIManager(): boolean { - if (!this.isDatabaseInitialized()) { - log.info("AI check failed: Database is not initialized"); - return false; - } - - try { - const aiManager = new AIServiceManager(); - const isAvailable = aiManager.isAnyServiceAvailable(); - log.info(`AI service availability check result: ${isAvailable}`); - return isAvailable; - } catch (error) { - log.error(`Error accessing AI service manager: ${error}`); - return false; - } - } /** * Handle a message sent to an LLM and get a response @@ -93,10 +74,14 @@ class RestChatService { return { error: "AI features are disabled. Please enable them in the settings." }; } - if (!this.safelyUseAIManager()) { - return { error: "AI services are currently unavailable. Please check your configuration." }; + // Check database initialization first + if (!this.isDatabaseInitialized()) { + throw new Error("Database is not initialized"); } + // Get or create AI service - will throw meaningful error if not possible + await aiServiceManager.getOrCreateAnyService(); + // Load or create chat directly from storage let chat = await chatStorageService.getChat(chatNoteId); diff --git a/apps/server/src/services/llm/config/configuration_manager.ts b/apps/server/src/services/llm/config/configuration_manager.ts index 7eaa435b1..1e082db41 100644 --- a/apps/server/src/services/llm/config/configuration_manager.ts +++ b/apps/server/src/services/llm/config/configuration_manager.ts @@ -70,7 +70,7 @@ export class ConfigurationManager { */ public async getSelectedProvider(): Promise { try { - const selectedProvider = await options.getOption('aiSelectedProvider'); + const selectedProvider = options.getOption('aiSelectedProvider'); return selectedProvider as ProviderType || null; } catch (error) { log.error(`Error getting selected provider: ${error}`); @@ -83,7 +83,7 @@ export class ConfigurationManager { */ public async getSelectedEmbeddingProvider(): Promise { try { - const selectedProvider = await options.getOption('embeddingSelectedProvider'); + const selectedProvider = options.getOption('embeddingSelectedProvider'); return selectedProvider as EmbeddingProviderType || null; } catch (error) { log.error(`Error getting selected embedding provider: ${error}`); @@ -155,11 +155,9 @@ export class ConfigurationManager { */ public async getDefaultModels(): Promise> { try { - const [openaiModel, anthropicModel, ollamaModel] = await Promise.all([ - options.getOption('openaiDefaultModel'), - options.getOption('anthropicDefaultModel'), - options.getOption('ollamaDefaultModel') - ]); + const openaiModel = options.getOption('openaiDefaultModel'); + const anthropicModel = options.getOption('anthropicDefaultModel'); + const ollamaModel = options.getOption('ollamaDefaultModel'); return { openai: openaiModel || undefined, @@ -182,20 +180,14 @@ export class ConfigurationManager { */ public async getProviderSettings(): Promise { try { - const [ - openaiApiKey, openaiBaseUrl, openaiDefaultModel, - anthropicApiKey, anthropicBaseUrl, anthropicDefaultModel, - ollamaBaseUrl, ollamaDefaultModel - ] = await Promise.all([ - options.getOption('openaiApiKey'), - options.getOption('openaiBaseUrl'), - options.getOption('openaiDefaultModel'), - options.getOption('anthropicApiKey'), - options.getOption('anthropicBaseUrl'), - options.getOption('anthropicDefaultModel'), - options.getOption('ollamaBaseUrl'), - options.getOption('ollamaDefaultModel') - ]); + const openaiApiKey = options.getOption('openaiApiKey'); + const openaiBaseUrl = options.getOption('openaiBaseUrl'); + const openaiDefaultModel = options.getOption('openaiDefaultModel'); + const anthropicApiKey = options.getOption('anthropicApiKey'); + const anthropicBaseUrl = options.getOption('anthropicBaseUrl'); + const anthropicDefaultModel = options.getOption('anthropicDefaultModel'); + const ollamaBaseUrl = options.getOption('ollamaBaseUrl'); + const ollamaDefaultModel = options.getOption('ollamaDefaultModel'); const settings: ProviderSettings = {}; @@ -302,7 +294,7 @@ export class ConfigurationManager { private async getAIEnabled(): Promise { try { - return await options.getOptionBool('aiEnabled'); + return options.getOptionBool('aiEnabled'); } catch { return false; } From cb3844e6277211283ab6c119e2e259ad7754f06f Mon Sep 17 00:00:00 2001 From: perf3ct Date: Sat, 7 Jun 2025 00:27:56 +0000 Subject: [PATCH 16/18] fix(llm): fix duplicated text when streaming responses --- .../services/llm/chat/rest_chat_service.ts | 12 ++--------- .../services/llm/pipeline/chat_pipeline.ts | 20 ++++++++++++++++--- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/apps/server/src/services/llm/chat/rest_chat_service.ts b/apps/server/src/services/llm/chat/rest_chat_service.ts index 1d5434ac1..53ea457a1 100644 --- a/apps/server/src/services/llm/chat/rest_chat_service.ts +++ b/apps/server/src/services/llm/chat/rest_chat_service.ts @@ -237,14 +237,6 @@ class RestChatService { // Send WebSocket message wsService.sendMessageToAllClients(message); - - // Send SSE response for compatibility - const responseData: any = { content: data, done }; - if (rawChunk?.toolExecution) { - responseData.toolExecution = rawChunk.toolExecution; - } - - res.write(`data: ${JSON.stringify(responseData)}\n\n`); // When streaming is complete, save the accumulated content to the chat note if (done) { @@ -266,8 +258,8 @@ class RestChatService { log.error(`Error saving streaming response: ${error}`); } - // End the response - res.end(); + // Note: For WebSocket-only streaming, we don't end the HTTP response here + // since it was already handled by the calling endpoint } } diff --git a/apps/server/src/services/llm/pipeline/chat_pipeline.ts b/apps/server/src/services/llm/pipeline/chat_pipeline.ts index 947a562e6..50671a809 100644 --- a/apps/server/src/services/llm/pipeline/chat_pipeline.ts +++ b/apps/server/src/services/llm/pipeline/chat_pipeline.ts @@ -298,6 +298,9 @@ export class ChatPipeline { this.updateStageMetrics('llmCompletion', llmStartTime); log.info(`Received LLM response from model: ${completion.response.model}, provider: ${completion.response.provider}`); + // Track whether content has been streamed to prevent duplication + let hasStreamedContent = false; + // Handle streaming if enabled and available // Use shouldEnableStream variable which contains our streaming decision if (shouldEnableStream && completion.response.stream && streamCallback) { @@ -311,6 +314,9 @@ export class ChatPipeline { // Forward to callback with original chunk data in case it contains additional information streamCallback(processedChunk.text, processedChunk.done, chunk); + + // Mark that we have streamed content to prevent duplication + hasStreamedContent = true; }); } @@ -767,11 +773,15 @@ export class ChatPipeline { const responseText = currentResponse.text || ""; log.info(`Resuming streaming with final response: ${responseText.length} chars`); - if (responseText.length > 0) { - // Resume streaming with the final response text + if (responseText.length > 0 && !hasStreamedContent) { + // Resume streaming with the final response text only if we haven't already streamed content // This is where we send the definitive done:true signal with the complete content streamCallback(responseText, true); log.info(`Sent final response with done=true signal and text content`); + } else if (hasStreamedContent) { + log.info(`Content already streamed, sending done=true signal only after tool execution`); + // Just send the done signal without duplicating content + streamCallback('', true); } else { // For Anthropic, sometimes text is empty but response is in stream if ((currentResponse.provider === 'Anthropic' || currentResponse.provider === 'OpenAI') && currentResponse.stream) { @@ -803,13 +813,17 @@ export class ChatPipeline { log.info(`LLM response did not contain any tool calls, skipping tool execution`); // Handle streaming for responses without tool calls - if (shouldEnableStream && streamCallback) { + if (shouldEnableStream && streamCallback && !hasStreamedContent) { log.info(`Sending final streaming response without tool calls: ${currentResponse.text.length} chars`); // Send the final response with done=true to complete the streaming streamCallback(currentResponse.text, true); log.info(`Sent final non-tool response with done=true signal`); + } else if (shouldEnableStream && streamCallback && hasStreamedContent) { + log.info(`Content already streamed, sending done=true signal only`); + // Just send the done signal without duplicating content + streamCallback('', true); } } From 4732d7784f16e1d100697fb7b812e1907a76a5d0 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Sat, 7 Jun 2025 04:13:57 +0000 Subject: [PATCH 17/18] fix(llm): add missing translations --- .../src/translations/en/translation.json | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/apps/client/src/translations/en/translation.json b/apps/client/src/translations/en/translation.json index 4d39265d1..f895587ff 100644 --- a/apps/client/src/translations/en/translation.json +++ b/apps/client/src/translations/en/translation.json @@ -1314,7 +1314,25 @@ "create_new_ai_chat": "Create new AI Chat" }, "create_new_ai_chat": "Create new AI Chat", - "configuration_warnings": "There are some issues with your AI configuration. Please check your settings." + "configuration_warnings": "There are some issues with your AI configuration. Please check your settings.", + "embeddings_started": "Embedding generation started", + "embeddings_stopped": "Embedding generation stopped", + "embeddings_toggle_error": "Error toggling embeddings", + "local_embedding_description": "Uses local embedding models for offline text embedding generation", + "local_embedding_settings": "Local Embedding Settings", + "ollama_embedding_settings": "Ollama Embedding Settings", + "ollama_embedding_url_description": "URL for the Ollama API for embedding generation (default: http://localhost:11434)", + "openai_embedding_api_key_description": "Your OpenAI API key for embedding generation (can be different from chat API key)", + "openai_embedding_settings": "OpenAI Embedding Settings", + "openai_embedding_url_description": "Base URL for OpenAI embedding API (default: https://api.openai.com/v1)", + "selected_embedding_provider": "Selected Embedding Provider", + "selected_embedding_provider_description": "Choose the provider for generating note embeddings", + "selected_provider": "Selected Provider", + "selected_provider_description": "Choose the AI provider for chat and completion features", + "select_embedding_provider": "Select embedding provider...", + "select_model": "Select model...", + "select_provider": "Select provider...", + "voyage_embedding_url_description": "Base URL for the Voyage AI embedding API (default: https://api.voyageai.com/v1)" }, "zoom_factor": { "title": "Zoom Factor (desktop build only)", From 6fdd0d021cfe3944524e5979f803bff38e67ec30 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Sat, 7 Jun 2025 04:23:18 +0000 Subject: [PATCH 18/18] fix(llm): don't show embedding models in the chat section --- .../options/ai_settings/ai_settings_widget.ts | 2 -- .../type_widgets/options/ai_settings/template.ts | 16 ---------------- 2 files changed, 18 deletions(-) diff --git a/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts b/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts index ec6e3e576..911726883 100644 --- a/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts +++ b/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts @@ -118,7 +118,6 @@ export default class AiSettingsWidget extends OptionsWidget { this.setupChangeHandler('.ollama-base-url', 'ollamaBaseUrl'); this.setupChangeHandler('.ollama-default-model', 'ollamaDefaultModel'); this.setupChangeHandler('.ollama-embedding-model', 'ollamaEmbeddingModel'); - this.setupChangeHandler('.ollama-chat-embedding-model', 'ollamaEmbeddingModel'); this.setupChangeHandler('.ollama-embedding-base-url', 'ollamaEmbeddingBaseUrl'); // Embedding-specific provider options @@ -671,7 +670,6 @@ export default class AiSettingsWidget extends OptionsWidget { this.$widget.find('.ollama-embedding-base-url').val(options.ollamaEmbeddingBaseUrl || 'http://localhost:11434'); this.setModelDropdownValue('.ollama-default-model', options.ollamaDefaultModel); this.setModelDropdownValue('.ollama-embedding-model', options.ollamaEmbeddingModel); - this.setModelDropdownValue('.ollama-chat-embedding-model', options.ollamaEmbeddingModel); // Embedding-specific provider options this.$widget.find('.openai-embedding-api-key').val(options.openaiEmbeddingApiKey || ''); diff --git a/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts b/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts index ac09f6b97..25b014dc7 100644 --- a/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts +++ b/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts @@ -98,14 +98,6 @@ export const TPL = `
${t("ai_llm.openai_model_description")}
- -
- - -
${t("ai_llm.openai_embedding_model_description")}
-
@@ -162,14 +154,6 @@ export const TPL = `
${t("ai_llm.ollama_model_description")}
- -
- - -
${t("ai_llm.ollama_embedding_model_description")}
-