Tools/Notes
1
0
Fork 0
mirror of https://github.com/TriliumNext/Notes.git synced 2025-10-31 04:51:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(llm): transition from initializing LLM providers, to creating them on demand

Browse Source
This commit is contained in:
perf3ct 2025-06-05 19:27:45 +00:00
parent c1b10d70b8
commit bb8a374ab8
No known key found for this signature in database
GPG Key ID: 569C4EEC436F5232
12 changed files with 246 additions and 261 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

197
apps/server/src/services/llm/ai_service_manager.ts
View File

@ -43,11 +43,7 @@ interface NoteContext {
}
export class AIServiceManager implements IAIServiceManager {
private services: Record<ServiceProviders, AIService> = {
openai: new OpenAIService(),
anthropic: new AnthropicService(),
ollama: new OllamaService()
};
private services: Partial<Record<ServiceProviders, AIService>> = {};
private providerOrder: ServiceProviders[] = []; // Will be populated from configuration
private initialized = false;
@ -183,9 +179,42 @@ export class AIServiceManager implements IAIServiceManager {
*/
getAvailableProviders(): ServiceProviders[] {
this.ensureInitialized();
return Object.entries(this.services)
.filter(([_, service]) => service.isAvailable())
.map(([key, _]) => key as ServiceProviders);
const allProviders: ServiceProviders[] = ['openai', 'anthropic', 'ollama'];
const availableProviders: ServiceProviders[] = [];
for (const providerName of allProviders) {
// Use a sync approach - check if we can create the provider
const service = this.services[providerName];
if (service && service.isAvailable()) {
availableProviders.push(providerName);
} else {
// For providers not yet created, check configuration to see if they would be available
try {
switch (providerName) {
case 'openai':
if (options.getOption('openaiApiKey')) {
availableProviders.push(providerName);
}
break;
case 'anthropic':
if (options.getOption('anthropicApiKey')) {
availableProviders.push(providerName);
}
break;
case 'ollama':
if (options.getOption('ollamaBaseUrl')) {
availableProviders.push(providerName);
}
break;
}
} catch (error) {
// Ignore configuration errors, provider just won't be available
}
}
}
return availableProviders;
}
/**
@ -224,9 +253,12 @@ export class AIServiceManager implements IAIServiceManager {
if (modelIdentifier.provider && availableProviders.includes(modelIdentifier.provider as ServiceProviders)) {
try {
const modifiedOptions = { ...options, model: modelIdentifier.modelId };
log.info(`[AIServiceManager] Using provider ${modelIdentifier.provider} from model prefix with modifiedOptions.stream: ${modifiedOptions.stream}`);
return await this.services[modelIdentifier.provider as ServiceProviders].generateChatCompletion(messages, modifiedOptions);
const service = this.services[modelIdentifier.provider as ServiceProviders];
if (service) {
const modifiedOptions = { ...options, model: modelIdentifier.modelId };
log.info(`[AIServiceManager] Using provider ${modelIdentifier.provider} from model prefix with modifiedOptions.stream: ${modifiedOptions.stream}`);
return await service.generateChatCompletion(messages, modifiedOptions);
}
} catch (error) {
log.error(`Error with specified provider ${modelIdentifier.provider}: ${error}`);
// If the specified provider fails, continue with the fallback providers
@ -240,8 +272,11 @@ export class AIServiceManager implements IAIServiceManager {
for (const provider of sortedProviders) {
try {
log.info(`[AIServiceManager] Trying provider ${provider} with options.stream: ${options.stream}`);
return await this.services[provider].generateChatCompletion(messages, options);
const service = this.services[provider];
if (service) {
log.info(`[AIServiceManager] Trying provider ${provider} with options.stream: ${options.stream}`);
return await service.generateChatCompletion(messages, options);
}
} catch (error) {
log.error(`Error with provider ${provider}: ${error}`);
lastError = error as Error;
@ -348,30 +383,49 @@ export class AIServiceManager implements IAIServiceManager {
}
/**
* Set up embeddings provider using the new configuration system
* Get or create a chat provider on-demand
*/
async setupEmbeddingsProvider(): Promise<void> {
try {
const aiEnabled = await isAIEnabled();
if (!aiEnabled) {
log.info('AI features are disabled');
return;
}
// Use the new configuration system - no string parsing!
const enabledProviders = await getEnabledEmbeddingProviders();
if (enabledProviders.length === 0) {
log.info('No embedding providers are enabled');
return;
}
// Initialize embedding providers
log.info('Embedding providers initialized successfully');
} catch (error: any) {
log.error(`Error setting up embedding providers: ${error.message}`);
throw error;
private async getOrCreateChatProvider(providerName: ServiceProviders): Promise<AIService | null> {
// Return existing provider if already created
if (this.services[providerName]) {
return this.services[providerName];
}
// Create provider on-demand based on configuration
try {
switch (providerName) {
case 'openai':
const openaiApiKey = await options.getOption('openaiApiKey');
if (openaiApiKey) {
this.services.openai = new OpenAIService();
log.info('Created OpenAI chat provider on-demand');
return this.services.openai;
}
break;
case 'anthropic':
const anthropicApiKey = await options.getOption('anthropicApiKey');
if (anthropicApiKey) {
this.services.anthropic = new AnthropicService();
log.info('Created Anthropic chat provider on-demand');
return this.services.anthropic;
}
break;
case 'ollama':
const ollamaBaseUrl = await options.getOption('ollamaBaseUrl');
if (ollamaBaseUrl) {
this.services.ollama = new OllamaService();
log.info('Created Ollama chat provider on-demand');
return this.services.ollama;
}
break;
}
} catch (error: any) {
log.error(`Error creating ${providerName} provider on-demand: ${error.message || 'Unknown error'}`);
}
return null;
}
/**
@ -392,9 +446,6 @@ export class AIServiceManager implements IAIServiceManager {
// Update provider order from configuration
await this.updateProviderOrderAsync();
// Set up embeddings provider if AI is enabled
await this.setupEmbeddingsProvider();
// Initialize index service
await this.getIndexService().initialize();
@ -462,7 +513,7 @@ export class AIServiceManager implements IAIServiceManager {
try {
// Get the default LLM service for context enhancement
const provider = this.getPreferredProvider();
const llmService = this.getService(provider);
const llmService = await this.getService(provider);
// Find relevant notes
contextNotes = await contextService.findRelevantNotes(
@ -503,25 +554,27 @@ export class AIServiceManager implements IAIServiceManager {
/**
* Get AI service for the given provider
*/
getService(provider?: string): AIService {
async getService(provider?: string): Promise<AIService> {
this.ensureInitialized();
// If provider is specified, try to use it
if (provider && this.services[provider as ServiceProviders]?.isAvailable()) {
return this.services[provider as ServiceProviders];
}
// Otherwise, use the first available provider in the configured order
for (const providerName of this.providerOrder) {
const service = this.services[providerName];
if (service.isAvailable()) {
// If provider is specified, try to get or create it
if (provider) {
const service = await this.getOrCreateChatProvider(provider as ServiceProviders);
if (service && service.isAvailable()) {
return service;
}
}
// If no provider is available, use first one anyway (it will throw an error)
// This allows us to show a proper error message rather than "provider not found"
return this.services[this.providerOrder[0]];
// Otherwise, try providers in the configured order
for (const providerName of this.providerOrder) {
const service = await this.getOrCreateChatProvider(providerName);
if (service && service.isAvailable()) {
return service;
}
}
// If no provider is available, throw a clear error
throw new Error('No AI chat providers are available. Please check your AI settings.');
}
/**
@ -550,7 +603,8 @@ export class AIServiceManager implements IAIServiceManager {
// Return the first available provider in the order
for (const providerName of this.providerOrder) {
if (this.services[providerName].isAvailable()) {
const service = this.services[providerName];
if (service && service.isAvailable()) {
return providerName;
}
}
@ -634,13 +688,15 @@ export class AIServiceManager implements IAIServiceManager {
// Initialize embeddings through index service
await indexService.startEmbeddingGeneration();
} else {
log.info('AI features disabled, stopping embeddings');
log.info('AI features disabled, stopping embeddings and clearing providers');
// Stop embeddings through index service
await indexService.stopEmbeddingGeneration();
// Clear chat providers
this.services = {};
}
} else {
// For other AI-related options, just recreate services
this.recreateServices();
// For other AI-related options, recreate services on-demand
await this.recreateServices();
}
}
});
@ -656,8 +712,12 @@ export class AIServiceManager implements IAIServiceManager {
// Clear configuration cache first
clearConfigurationCache();
// Recreate all service instances to pick up new configuration
this.recreateServiceInstances();
// Clear existing chat providers (they will be recreated on-demand)
this.services = {};
// Clear embedding providers (they will be recreated on-demand when needed)
const providerManager = await import('./providers/providers.js');
providerManager.clearAllEmbeddingProviders();
// Update provider order with new configuration
await this.updateProviderOrderAsync();
@ -668,25 +728,6 @@ export class AIServiceManager implements IAIServiceManager {
}
}
/**
* Recreate service instances to pick up new configuration
*/
private recreateServiceInstances(): void {
try {
log.info('Recreating service instances');
// Recreate service instances
this.services = {
openai: new OpenAIService(),
anthropic: new AnthropicService(),
ollama: new OllamaService()
};
log.info('Service instances recreated successfully');
} catch (error) {
log.error(`Error recreating service instances: ${this.handleError(error)}`);
}
}
}
// Don't create singleton immediately, use a lazy-loading pattern
@ -759,7 +800,7 @@ export default {
);
},
// New methods
getService(provider?: string): AIService {
async getService(provider?: string): Promise<AIService> {
return getInstance().getService(provider);
},
getPreferredProvider(): string {

4
apps/server/src/services/llm/context/index.ts
View File

@ -33,7 +33,7 @@ async function getSemanticContext(
}
// Get an LLM service
const llmService = aiServiceManager.getInstance().getService();
const llmService = await aiServiceManager.getInstance().getService();
const result = await contextService.processQuery("", llmService, {
maxResults: options.maxSimilarNotes || 5,
@ -543,7 +543,7 @@ export class ContextExtractor {
try {
const { default: aiServiceManager } = await import('../ai_service_manager.js');
const contextService = aiServiceManager.getInstance().getContextService();
const llmService = aiServiceManager.getInstance().getService();
const llmService = await aiServiceManager.getInstance().getService();
if (!contextService) {
return "Context service not available.";

3
apps/server/src/services/llm/embeddings/init.ts
View File

@ -45,8 +45,7 @@ export async function initializeEmbeddings() {
// Start the embedding system if AI is enabled
if (await options.getOptionBool('aiEnabled')) {
// Initialize default embedding providers when AI is enabled
await providerManager.initializeDefaultProviders();
// Embedding providers will be created on-demand when needed
await initEmbeddings();
log.info("Embedding system initialized successfully.");
} else {

11
apps/server/src/services/llm/index_service.ts
View File

@ -851,10 +851,6 @@ export class IndexService {
throw new Error("AI features must be enabled first");
}
// Re-initialize providers first in case they weren't available when server started
log.info("Re-initializing embedding providers");
await providerManager.initializeDefaultProviders();
// Re-initialize if needed
if (!this.initialized) {
await this.initialize();
@ -870,6 +866,13 @@ export class IndexService {
return;
}
// Verify providers are available (this will create them on-demand if needed)
const providers = await providerManager.getEnabledEmbeddingProviders();
if (providers.length === 0) {
throw new Error("No embedding providers available");
}
log.info(`Found ${providers.length} embedding providers: ${providers.map(p => p.name).join(', ')}`);
// Setup automatic indexing if enabled
if (await options.getOptionBool('embeddingAutoUpdateEnabled')) {
this.setupAutomaticIndexing();

2
apps/server/src/services/llm/interfaces/ai_service_interfaces.ts
View File

@ -28,7 +28,7 @@ export interface AIServiceManagerConfig {
* Interface for managing AI service providers
*/
export interface IAIServiceManager {
getService(provider?: string): AIService;
getService(provider?: string): Promise<AIService>;
getAvailableProviders(): string[];
getPreferredProvider(): string;
isProviderAvailable(provider: string): boolean;

2
apps/server/src/services/llm/pipeline/stages/context_extraction_stage.ts
View File

@ -43,7 +43,7 @@ export class ContextExtractionStage {
// Get enhanced context from the context service
const contextService = aiServiceManager.getContextService();
const llmService = aiServiceManager.getService();
const llmService = await aiServiceManager.getService();
if (contextService) {
// Use unified context service to get smart context

2
apps/server/src/services/llm/pipeline/stages/llm_completion_stage.ts
View File

@ -104,7 +104,7 @@ export class LLMCompletionStage extends BasePipelineStage<LLMCompletionInput, {
// Use specific provider if available
if (selectedProvider && aiServiceManager.isProviderAvailable(selectedProvider)) {
const service = aiServiceManager.getService(selectedProvider);
const service = await aiServiceManager.getService(selectedProvider);
log.info(`[LLMCompletionStage] Using specific service for ${selectedProvider}`);
// Generate completion and wrap with enhanced stream handling

2
apps/server/src/services/llm/pipeline/stages/model_selection_stage.ts
View File

@ -292,7 +292,7 @@ export class ModelSelectionStage extends BasePipelineStage<ModelSelectionInput,
log.info(`Getting default model for provider ${provider} using AI service manager`);
// Use the existing AI service manager instead of duplicating API calls
const service = aiServiceManager.getInstance().getService(provider);
const service = await aiServiceManager.getInstance().getService(provider);
if (!service || !service.isAvailable()) {
log.info(`Provider ${provider} service is not available`);

247
apps/server/src/services/llm/providers/providers.ts
View File

@ -123,6 +123,94 @@ export function getEmbeddingProvider(name: string): EmbeddingProvider | undefine
return providers.get(name);
}
/**
* Create providers on-demand based on current options
*/
export async function createProvidersFromCurrentOptions(): Promise<EmbeddingProvider[]> {
const result: EmbeddingProvider[] = [];
try {
// Create Ollama provider if embedding base URL is configured
const ollamaEmbeddingBaseUrl = await options.getOption('ollamaEmbeddingBaseUrl');
if (ollamaEmbeddingBaseUrl) {
const embeddingModel = await options.getOption('ollamaEmbeddingModel');
try {
const ollamaProvider = new OllamaEmbeddingProvider({
model: embeddingModel,
dimension: 768, // Initial value, will be updated during initialization
type: 'float32',
baseUrl: ollamaEmbeddingBaseUrl
});
await ollamaProvider.initialize();
registerEmbeddingProvider(ollamaProvider);
result.push(ollamaProvider);
log.info(`Created Ollama provider on-demand: ${embeddingModel} at ${ollamaEmbeddingBaseUrl}`);
} catch (error: any) {
log.error(`Error creating Ollama embedding provider on-demand: ${error.message || 'Unknown error'}`);
}
}
// Create OpenAI provider if API key is configured
const openaiApiKey = await options.getOption('openaiApiKey');
if (openaiApiKey) {
const openaiModel = await options.getOption('openaiEmbeddingModel') || 'text-embedding-3-small';
const openaiBaseUrl = await options.getOption('openaiBaseUrl') || 'https://api.openai.com/v1';
const openaiProvider = new OpenAIEmbeddingProvider({
model: openaiModel,
dimension: 1536,
type: 'float32',
apiKey: openaiApiKey,
baseUrl: openaiBaseUrl
});
registerEmbeddingProvider(openaiProvider);
result.push(openaiProvider);
log.info(`Created OpenAI provider on-demand: ${openaiModel}`);
}
// Create Voyage provider if API key is configured
const voyageApiKey = await options.getOption('voyageApiKey' as any);
if (voyageApiKey) {
const voyageModel = await options.getOption('voyageEmbeddingModel') || 'voyage-2';
const voyageBaseUrl = 'https://api.voyageai.com/v1';
const voyageProvider = new VoyageEmbeddingProvider({
model: voyageModel,
dimension: 1024,
type: 'float32',
apiKey: voyageApiKey,
baseUrl: voyageBaseUrl
});
registerEmbeddingProvider(voyageProvider);
result.push(voyageProvider);
log.info(`Created Voyage provider on-demand: ${voyageModel}`);
}
// Always include local provider as fallback
if (!providers.has('local')) {
const localProvider = new SimpleLocalEmbeddingProvider({
model: 'local',
dimension: 384,
type: 'float32'
});
registerEmbeddingProvider(localProvider);
result.push(localProvider);
log.info(`Created local provider on-demand as fallback`);
} else {
result.push(providers.get('local')!);
}
} catch (error: any) {
log.error(`Error creating providers from current options: ${error.message || 'Unknown error'}`);
}
return result;
}
/**
* Get all enabled embedding providers
*/
@ -131,31 +219,16 @@ export async function getEnabledEmbeddingProviders(): Promise<EmbeddingProvider[
return [];
}
// Get providers from database ordered by priority
const dbProviders = await sql.getRows(`
SELECT providerId, name, config
FROM embedding_providers
ORDER BY priority DESC`
);
const result: EmbeddingProvider[] = [];
for (const row of dbProviders) {
const rowData = row as any;
const provider = providers.get(rowData.name);
if (provider) {
result.push(provider);
} else {
// Only log error if we haven't logged it before for this provider
if (!loggedProviderErrors.has(rowData.name)) {
log.error(`Enabled embedding provider ${rowData.name} not found in registered providers`);
loggedProviderErrors.add(rowData.name);
}
}
// First try to get existing registered providers
const existingProviders = Array.from(providers.values());
// If no providers are registered, create them on-demand from current options
if (existingProviders.length === 0) {
log.info('No providers registered, creating from current options');
return await createProvidersFromCurrentOptions();
}
return result;
return existingProviders;
}
/**
@ -257,130 +330,13 @@ export async function getEmbeddingProviderConfigs() {
/**
* Initialize the default embedding providers
* @deprecated - Use on-demand provider creation instead
*/
export async function initializeDefaultProviders() {
// Register built-in providers
try {
// Register OpenAI provider if API key is configured
const openaiApiKey = await options.getOption('openaiApiKey');
if (openaiApiKey) {
const openaiModel = await options.getOption('openaiEmbeddingModel') || 'text-embedding-3-small';
const openaiBaseUrl = await options.getOption('openaiBaseUrl') || 'https://api.openai.com/v1';
registerEmbeddingProvider(new OpenAIEmbeddingProvider({
model: openaiModel,
dimension: 1536, // OpenAI's typical dimension
type: 'float32',
apiKey: openaiApiKey,
baseUrl: openaiBaseUrl
}));
// Create OpenAI provider config if it doesn't exist
const existingOpenAI = await sql.getRow(
"SELECT * FROM embedding_providers WHERE name = ?",
['openai']
);
if (!existingOpenAI) {
await createEmbeddingProviderConfig('openai', {
model: openaiModel,
dimension: 1536,
type: 'float32'
}, 100);
}
}
// Register Voyage provider if API key is configured
const voyageApiKey = await options.getOption('voyageApiKey' as any);
if (voyageApiKey) {
const voyageModel = await options.getOption('voyageEmbeddingModel') || 'voyage-2';
const voyageBaseUrl = 'https://api.voyageai.com/v1';
registerEmbeddingProvider(new VoyageEmbeddingProvider({
model: voyageModel,
dimension: 1024, // Voyage's embedding dimension
type: 'float32',
apiKey: voyageApiKey,
baseUrl: voyageBaseUrl
}));
// Create Voyage provider config if it doesn't exist
const existingVoyage = await sql.getRow(
"SELECT * FROM embedding_providers WHERE name = ?",
['voyage']
);
if (!existingVoyage) {
await createEmbeddingProviderConfig('voyage', {
model: voyageModel,
dimension: 1024,
type: 'float32'
}, 75);
}
}
// Register Ollama embedding provider if embedding base URL is configured
const ollamaEmbeddingBaseUrl = await options.getOption('ollamaEmbeddingBaseUrl');
if (ollamaEmbeddingBaseUrl) {
// Use specific embedding models if available
const embeddingModel = await options.getOption('ollamaEmbeddingModel');
try {
// Create provider with initial dimension to be updated during initialization
const ollamaProvider = new OllamaEmbeddingProvider({
model: embeddingModel,
dimension: 768, // Initial value, will be updated during initialization
type: 'float32',
baseUrl: ollamaEmbeddingBaseUrl
});
// Register the provider
registerEmbeddingProvider(ollamaProvider);
// Initialize the provider to detect model capabilities
await ollamaProvider.initialize();
// Create Ollama provider config if it doesn't exist
const existingOllama = await sql.getRow(
"SELECT * FROM embedding_providers WHERE name = ?",
['ollama']
);
if (!existingOllama) {
await createEmbeddingProviderConfig('ollama', {
model: embeddingModel,
dimension: ollamaProvider.getDimension(),
type: 'float32'
}, 50);
}
} catch (error: any) {
log.error(`Error initializing Ollama embedding provider: ${error.message || 'Unknown error'}`);
}
}
// Always register local provider as fallback
registerEmbeddingProvider(new SimpleLocalEmbeddingProvider({
model: 'local',
dimension: 384,
type: 'float32'
}));
// Create local provider config if it doesn't exist
const existingLocal = await sql.getRow(
"SELECT * FROM embedding_providers WHERE name = ?",
['local']
);
if (!existingLocal) {
await createEmbeddingProviderConfig('local', {
model: 'local',
dimension: 384,
type: 'float32'
}, 10);
}
} catch (error: any) {
log.error(`Error initializing default embedding providers: ${error.message || 'Unknown error'}`);
}
// This function is now deprecated in favor of on-demand provider creation
// The createProvidersFromCurrentOptions() function should be used instead
log.info('initializeDefaultProviders called - using on-demand provider creation instead');
return await createProvidersFromCurrentOptions();
}
export default {
@ -390,6 +346,7 @@ export default {
getEmbeddingProviders,
getEmbeddingProvider,
getEnabledEmbeddingProviders,
createProvidersFromCurrentOptions,
createEmbeddingProviderConfig,
updateEmbeddingProviderConfig,
deleteEmbeddingProviderConfig,

7
apps/server/src/services/llm/tools/note_summarization_tool.ts
View File

@ -102,12 +102,7 @@ export class NoteSummarizationTool implements ToolHandler {
const cleanContent = this.cleanHtml(content);
// Generate the summary using the AI service
const aiService = aiServiceManager.getService();
if (!aiService) {
log.error('No AI service available for summarization');
return `Error: No AI service is available for summarization`;
}
const aiService = await aiServiceManager.getService();
log.info(`Using ${aiService.getName()} to generate summary`);

11
apps/server/src/services/llm/tools/relationship_tool.ts
View File

@ -312,16 +312,7 @@ export class RelationshipTool implements ToolHandler {
}
// Get the AI service for relationship suggestion
const aiService = aiServiceManager.getService();
if (!aiService) {
log.error('No AI service available for relationship suggestions');
return {
success: false,
message: 'AI service not available for relationship suggestions',
relatedNotes: relatedResult.relatedNotes
};
}
const aiService = await aiServiceManager.getService();
log.info(`Using ${aiService.getName()} to suggest relationships for ${relatedResult.relatedNotes.length} related notes`);

19
apps/server/src/services/llm/tools/search_notes_tool.ts
View File

@ -122,10 +122,10 @@ export class SearchNotesTool implements ToolHandler {
// If summarization is requested
if (summarize) {
// Try to get an LLM service for summarization
const llmService = aiServiceManager.getService();
if (llmService) {
try {
const messages = [
try {
const llmService = await aiServiceManager.getService();
const messages = [
{
role: "system" as const,
content: "Summarize the following note content concisely while preserving key information. Keep your summary to about 3-4 sentences."
@ -147,13 +147,12 @@ export class SearchNotesTool implements ToolHandler {
} as Record<string, boolean>))
});
if (result && result.text) {
return result.text;
}
} catch (error) {
log.error(`Error summarizing content: ${error}`);
// Fall through to smart truncation if summarization fails
if (result && result.text) {
return result.text;
}
} catch (error) {
log.error(`Error summarizing content: ${error}`);
// Fall through to smart truncation if summarization fails
}
}