From bb8a374ab8a2655f5f8774aa40f08eb8630f81a2 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Thu, 5 Jun 2025 19:27:45 +0000 Subject: [PATCH] feat(llm): transition from initializing LLM providers, to creating them on demand --- .../src/services/llm/ai_service_manager.ts | 197 ++++++++------ apps/server/src/services/llm/context/index.ts | 4 +- .../src/services/llm/embeddings/init.ts | 3 +- apps/server/src/services/llm/index_service.ts | 11 +- .../llm/interfaces/ai_service_interfaces.ts | 2 +- .../stages/context_extraction_stage.ts | 2 +- .../pipeline/stages/llm_completion_stage.ts | 2 +- .../pipeline/stages/model_selection_stage.ts | 2 +- .../src/services/llm/providers/providers.ts | 247 ++++++++---------- .../llm/tools/note_summarization_tool.ts | 7 +- .../services/llm/tools/relationship_tool.ts | 11 +- .../services/llm/tools/search_notes_tool.ts | 19 +- 12 files changed, 246 insertions(+), 261 deletions(-) diff --git a/apps/server/src/services/llm/ai_service_manager.ts b/apps/server/src/services/llm/ai_service_manager.ts index 9e84d1e28..f054dff57 100644 --- a/apps/server/src/services/llm/ai_service_manager.ts +++ b/apps/server/src/services/llm/ai_service_manager.ts @@ -43,11 +43,7 @@ interface NoteContext { } export class AIServiceManager implements IAIServiceManager { - private services: Record = { - openai: new OpenAIService(), - anthropic: new AnthropicService(), - ollama: new OllamaService() - }; + private services: Partial> = {}; private providerOrder: ServiceProviders[] = []; // Will be populated from configuration private initialized = false; @@ -183,9 +179,42 @@ export class AIServiceManager implements IAIServiceManager { */ getAvailableProviders(): ServiceProviders[] { this.ensureInitialized(); - return Object.entries(this.services) - .filter(([_, service]) => service.isAvailable()) - .map(([key, _]) => key as ServiceProviders); + + const allProviders: ServiceProviders[] = ['openai', 'anthropic', 'ollama']; + const availableProviders: ServiceProviders[] = []; + + for (const providerName of allProviders) { + // Use a sync approach - check if we can create the provider + const service = this.services[providerName]; + if (service && service.isAvailable()) { + availableProviders.push(providerName); + } else { + // For providers not yet created, check configuration to see if they would be available + try { + switch (providerName) { + case 'openai': + if (options.getOption('openaiApiKey')) { + availableProviders.push(providerName); + } + break; + case 'anthropic': + if (options.getOption('anthropicApiKey')) { + availableProviders.push(providerName); + } + break; + case 'ollama': + if (options.getOption('ollamaBaseUrl')) { + availableProviders.push(providerName); + } + break; + } + } catch (error) { + // Ignore configuration errors, provider just won't be available + } + } + } + + return availableProviders; } /** @@ -224,9 +253,12 @@ export class AIServiceManager implements IAIServiceManager { if (modelIdentifier.provider && availableProviders.includes(modelIdentifier.provider as ServiceProviders)) { try { - const modifiedOptions = { ...options, model: modelIdentifier.modelId }; - log.info(`[AIServiceManager] Using provider ${modelIdentifier.provider} from model prefix with modifiedOptions.stream: ${modifiedOptions.stream}`); - return await this.services[modelIdentifier.provider as ServiceProviders].generateChatCompletion(messages, modifiedOptions); + const service = this.services[modelIdentifier.provider as ServiceProviders]; + if (service) { + const modifiedOptions = { ...options, model: modelIdentifier.modelId }; + log.info(`[AIServiceManager] Using provider ${modelIdentifier.provider} from model prefix with modifiedOptions.stream: ${modifiedOptions.stream}`); + return await service.generateChatCompletion(messages, modifiedOptions); + } } catch (error) { log.error(`Error with specified provider ${modelIdentifier.provider}: ${error}`); // If the specified provider fails, continue with the fallback providers @@ -240,8 +272,11 @@ export class AIServiceManager implements IAIServiceManager { for (const provider of sortedProviders) { try { - log.info(`[AIServiceManager] Trying provider ${provider} with options.stream: ${options.stream}`); - return await this.services[provider].generateChatCompletion(messages, options); + const service = this.services[provider]; + if (service) { + log.info(`[AIServiceManager] Trying provider ${provider} with options.stream: ${options.stream}`); + return await service.generateChatCompletion(messages, options); + } } catch (error) { log.error(`Error with provider ${provider}: ${error}`); lastError = error as Error; @@ -348,30 +383,49 @@ export class AIServiceManager implements IAIServiceManager { } /** - * Set up embeddings provider using the new configuration system + * Get or create a chat provider on-demand */ - async setupEmbeddingsProvider(): Promise { - try { - const aiEnabled = await isAIEnabled(); - if (!aiEnabled) { - log.info('AI features are disabled'); - return; - } - - // Use the new configuration system - no string parsing! - const enabledProviders = await getEnabledEmbeddingProviders(); - - if (enabledProviders.length === 0) { - log.info('No embedding providers are enabled'); - return; - } - - // Initialize embedding providers - log.info('Embedding providers initialized successfully'); - } catch (error: any) { - log.error(`Error setting up embedding providers: ${error.message}`); - throw error; + private async getOrCreateChatProvider(providerName: ServiceProviders): Promise { + // Return existing provider if already created + if (this.services[providerName]) { + return this.services[providerName]; } + + // Create provider on-demand based on configuration + try { + switch (providerName) { + case 'openai': + const openaiApiKey = await options.getOption('openaiApiKey'); + if (openaiApiKey) { + this.services.openai = new OpenAIService(); + log.info('Created OpenAI chat provider on-demand'); + return this.services.openai; + } + break; + + case 'anthropic': + const anthropicApiKey = await options.getOption('anthropicApiKey'); + if (anthropicApiKey) { + this.services.anthropic = new AnthropicService(); + log.info('Created Anthropic chat provider on-demand'); + return this.services.anthropic; + } + break; + + case 'ollama': + const ollamaBaseUrl = await options.getOption('ollamaBaseUrl'); + if (ollamaBaseUrl) { + this.services.ollama = new OllamaService(); + log.info('Created Ollama chat provider on-demand'); + return this.services.ollama; + } + break; + } + } catch (error: any) { + log.error(`Error creating ${providerName} provider on-demand: ${error.message || 'Unknown error'}`); + } + + return null; } /** @@ -392,9 +446,6 @@ export class AIServiceManager implements IAIServiceManager { // Update provider order from configuration await this.updateProviderOrderAsync(); - // Set up embeddings provider if AI is enabled - await this.setupEmbeddingsProvider(); - // Initialize index service await this.getIndexService().initialize(); @@ -462,7 +513,7 @@ export class AIServiceManager implements IAIServiceManager { try { // Get the default LLM service for context enhancement const provider = this.getPreferredProvider(); - const llmService = this.getService(provider); + const llmService = await this.getService(provider); // Find relevant notes contextNotes = await contextService.findRelevantNotes( @@ -503,25 +554,27 @@ export class AIServiceManager implements IAIServiceManager { /** * Get AI service for the given provider */ - getService(provider?: string): AIService { + async getService(provider?: string): Promise { this.ensureInitialized(); - // If provider is specified, try to use it - if (provider && this.services[provider as ServiceProviders]?.isAvailable()) { - return this.services[provider as ServiceProviders]; - } - - // Otherwise, use the first available provider in the configured order - for (const providerName of this.providerOrder) { - const service = this.services[providerName]; - if (service.isAvailable()) { + // If provider is specified, try to get or create it + if (provider) { + const service = await this.getOrCreateChatProvider(provider as ServiceProviders); + if (service && service.isAvailable()) { return service; } } - // If no provider is available, use first one anyway (it will throw an error) - // This allows us to show a proper error message rather than "provider not found" - return this.services[this.providerOrder[0]]; + // Otherwise, try providers in the configured order + for (const providerName of this.providerOrder) { + const service = await this.getOrCreateChatProvider(providerName); + if (service && service.isAvailable()) { + return service; + } + } + + // If no provider is available, throw a clear error + throw new Error('No AI chat providers are available. Please check your AI settings.'); } /** @@ -550,7 +603,8 @@ export class AIServiceManager implements IAIServiceManager { // Return the first available provider in the order for (const providerName of this.providerOrder) { - if (this.services[providerName].isAvailable()) { + const service = this.services[providerName]; + if (service && service.isAvailable()) { return providerName; } } @@ -634,13 +688,15 @@ export class AIServiceManager implements IAIServiceManager { // Initialize embeddings through index service await indexService.startEmbeddingGeneration(); } else { - log.info('AI features disabled, stopping embeddings'); + log.info('AI features disabled, stopping embeddings and clearing providers'); // Stop embeddings through index service await indexService.stopEmbeddingGeneration(); + // Clear chat providers + this.services = {}; } } else { - // For other AI-related options, just recreate services - this.recreateServices(); + // For other AI-related options, recreate services on-demand + await this.recreateServices(); } } }); @@ -656,8 +712,12 @@ export class AIServiceManager implements IAIServiceManager { // Clear configuration cache first clearConfigurationCache(); - // Recreate all service instances to pick up new configuration - this.recreateServiceInstances(); + // Clear existing chat providers (they will be recreated on-demand) + this.services = {}; + + // Clear embedding providers (they will be recreated on-demand when needed) + const providerManager = await import('./providers/providers.js'); + providerManager.clearAllEmbeddingProviders(); // Update provider order with new configuration await this.updateProviderOrderAsync(); @@ -668,25 +728,6 @@ export class AIServiceManager implements IAIServiceManager { } } - /** - * Recreate service instances to pick up new configuration - */ - private recreateServiceInstances(): void { - try { - log.info('Recreating service instances'); - - // Recreate service instances - this.services = { - openai: new OpenAIService(), - anthropic: new AnthropicService(), - ollama: new OllamaService() - }; - - log.info('Service instances recreated successfully'); - } catch (error) { - log.error(`Error recreating service instances: ${this.handleError(error)}`); - } - } } // Don't create singleton immediately, use a lazy-loading pattern @@ -759,7 +800,7 @@ export default { ); }, // New methods - getService(provider?: string): AIService { + async getService(provider?: string): Promise { return getInstance().getService(provider); }, getPreferredProvider(): string { diff --git a/apps/server/src/services/llm/context/index.ts b/apps/server/src/services/llm/context/index.ts index 258428705..c44eea9cb 100644 --- a/apps/server/src/services/llm/context/index.ts +++ b/apps/server/src/services/llm/context/index.ts @@ -33,7 +33,7 @@ async function getSemanticContext( } // Get an LLM service - const llmService = aiServiceManager.getInstance().getService(); + const llmService = await aiServiceManager.getInstance().getService(); const result = await contextService.processQuery("", llmService, { maxResults: options.maxSimilarNotes || 5, @@ -543,7 +543,7 @@ export class ContextExtractor { try { const { default: aiServiceManager } = await import('../ai_service_manager.js'); const contextService = aiServiceManager.getInstance().getContextService(); - const llmService = aiServiceManager.getInstance().getService(); + const llmService = await aiServiceManager.getInstance().getService(); if (!contextService) { return "Context service not available."; diff --git a/apps/server/src/services/llm/embeddings/init.ts b/apps/server/src/services/llm/embeddings/init.ts index 94188fa69..6ee4afe4b 100644 --- a/apps/server/src/services/llm/embeddings/init.ts +++ b/apps/server/src/services/llm/embeddings/init.ts @@ -45,8 +45,7 @@ export async function initializeEmbeddings() { // Start the embedding system if AI is enabled if (await options.getOptionBool('aiEnabled')) { - // Initialize default embedding providers when AI is enabled - await providerManager.initializeDefaultProviders(); + // Embedding providers will be created on-demand when needed await initEmbeddings(); log.info("Embedding system initialized successfully."); } else { diff --git a/apps/server/src/services/llm/index_service.ts b/apps/server/src/services/llm/index_service.ts index a179431ee..9d118e274 100644 --- a/apps/server/src/services/llm/index_service.ts +++ b/apps/server/src/services/llm/index_service.ts @@ -851,10 +851,6 @@ export class IndexService { throw new Error("AI features must be enabled first"); } - // Re-initialize providers first in case they weren't available when server started - log.info("Re-initializing embedding providers"); - await providerManager.initializeDefaultProviders(); - // Re-initialize if needed if (!this.initialized) { await this.initialize(); @@ -870,6 +866,13 @@ export class IndexService { return; } + // Verify providers are available (this will create them on-demand if needed) + const providers = await providerManager.getEnabledEmbeddingProviders(); + if (providers.length === 0) { + throw new Error("No embedding providers available"); + } + log.info(`Found ${providers.length} embedding providers: ${providers.map(p => p.name).join(', ')}`); + // Setup automatic indexing if enabled if (await options.getOptionBool('embeddingAutoUpdateEnabled')) { this.setupAutomaticIndexing(); diff --git a/apps/server/src/services/llm/interfaces/ai_service_interfaces.ts b/apps/server/src/services/llm/interfaces/ai_service_interfaces.ts index 3126691a4..4130a8d55 100644 --- a/apps/server/src/services/llm/interfaces/ai_service_interfaces.ts +++ b/apps/server/src/services/llm/interfaces/ai_service_interfaces.ts @@ -28,7 +28,7 @@ export interface AIServiceManagerConfig { * Interface for managing AI service providers */ export interface IAIServiceManager { - getService(provider?: string): AIService; + getService(provider?: string): Promise; getAvailableProviders(): string[]; getPreferredProvider(): string; isProviderAvailable(provider: string): boolean; diff --git a/apps/server/src/services/llm/pipeline/stages/context_extraction_stage.ts b/apps/server/src/services/llm/pipeline/stages/context_extraction_stage.ts index 95d7620e2..b1eaa69f9 100644 --- a/apps/server/src/services/llm/pipeline/stages/context_extraction_stage.ts +++ b/apps/server/src/services/llm/pipeline/stages/context_extraction_stage.ts @@ -43,7 +43,7 @@ export class ContextExtractionStage { // Get enhanced context from the context service const contextService = aiServiceManager.getContextService(); - const llmService = aiServiceManager.getService(); + const llmService = await aiServiceManager.getService(); if (contextService) { // Use unified context service to get smart context diff --git a/apps/server/src/services/llm/pipeline/stages/llm_completion_stage.ts b/apps/server/src/services/llm/pipeline/stages/llm_completion_stage.ts index 7dd6984c8..6354e4c59 100644 --- a/apps/server/src/services/llm/pipeline/stages/llm_completion_stage.ts +++ b/apps/server/src/services/llm/pipeline/stages/llm_completion_stage.ts @@ -104,7 +104,7 @@ export class LLMCompletionStage extends BasePipelineStage { + const result: EmbeddingProvider[] = []; + + try { + // Create Ollama provider if embedding base URL is configured + const ollamaEmbeddingBaseUrl = await options.getOption('ollamaEmbeddingBaseUrl'); + if (ollamaEmbeddingBaseUrl) { + const embeddingModel = await options.getOption('ollamaEmbeddingModel'); + + try { + const ollamaProvider = new OllamaEmbeddingProvider({ + model: embeddingModel, + dimension: 768, // Initial value, will be updated during initialization + type: 'float32', + baseUrl: ollamaEmbeddingBaseUrl + }); + + await ollamaProvider.initialize(); + registerEmbeddingProvider(ollamaProvider); + result.push(ollamaProvider); + log.info(`Created Ollama provider on-demand: ${embeddingModel} at ${ollamaEmbeddingBaseUrl}`); + } catch (error: any) { + log.error(`Error creating Ollama embedding provider on-demand: ${error.message || 'Unknown error'}`); + } + } + + // Create OpenAI provider if API key is configured + const openaiApiKey = await options.getOption('openaiApiKey'); + if (openaiApiKey) { + const openaiModel = await options.getOption('openaiEmbeddingModel') || 'text-embedding-3-small'; + const openaiBaseUrl = await options.getOption('openaiBaseUrl') || 'https://api.openai.com/v1'; + + const openaiProvider = new OpenAIEmbeddingProvider({ + model: openaiModel, + dimension: 1536, + type: 'float32', + apiKey: openaiApiKey, + baseUrl: openaiBaseUrl + }); + + registerEmbeddingProvider(openaiProvider); + result.push(openaiProvider); + log.info(`Created OpenAI provider on-demand: ${openaiModel}`); + } + + // Create Voyage provider if API key is configured + const voyageApiKey = await options.getOption('voyageApiKey' as any); + if (voyageApiKey) { + const voyageModel = await options.getOption('voyageEmbeddingModel') || 'voyage-2'; + const voyageBaseUrl = 'https://api.voyageai.com/v1'; + + const voyageProvider = new VoyageEmbeddingProvider({ + model: voyageModel, + dimension: 1024, + type: 'float32', + apiKey: voyageApiKey, + baseUrl: voyageBaseUrl + }); + + registerEmbeddingProvider(voyageProvider); + result.push(voyageProvider); + log.info(`Created Voyage provider on-demand: ${voyageModel}`); + } + + // Always include local provider as fallback + if (!providers.has('local')) { + const localProvider = new SimpleLocalEmbeddingProvider({ + model: 'local', + dimension: 384, + type: 'float32' + }); + registerEmbeddingProvider(localProvider); + result.push(localProvider); + log.info(`Created local provider on-demand as fallback`); + } else { + result.push(providers.get('local')!); + } + + } catch (error: any) { + log.error(`Error creating providers from current options: ${error.message || 'Unknown error'}`); + } + + return result; +} + /** * Get all enabled embedding providers */ @@ -131,31 +219,16 @@ export async function getEnabledEmbeddingProviders(): Promise)) }); - if (result && result.text) { - return result.text; - } - } catch (error) { - log.error(`Error summarizing content: ${error}`); - // Fall through to smart truncation if summarization fails + if (result && result.text) { + return result.text; } + } catch (error) { + log.error(`Error summarizing content: ${error}`); + // Fall through to smart truncation if summarization fails } }