From 35f78aede936a66f68c471168be4780454d53f2d Mon Sep 17 00:00:00 2001 From: perf3ct Date: Mon, 2 Jun 2025 00:56:19 +0000 Subject: [PATCH 01/18] feat(llm): redo chat storage, part 1 --- .../src/widgets/llm_chat/communication.ts | 77 +++++++--------- .../src/widgets/llm_chat/llm_chat_panel.ts | 89 +++++++++++-------- apps/client/src/widgets/llm_chat/types.ts | 6 +- .../src/widgets/type_widgets/ai_chat.ts | 10 ++- apps/server/src/routes/api/llm.ts | 7 +- .../services/llm/chat/rest_chat_service.ts | 57 +++++++++--- 6 files changed, 142 insertions(+), 104 deletions(-) diff --git a/apps/client/src/widgets/llm_chat/communication.ts b/apps/client/src/widgets/llm_chat/communication.ts index bb58a47b2..87687cab9 100644 --- a/apps/client/src/widgets/llm_chat/communication.ts +++ b/apps/client/src/widgets/llm_chat/communication.ts @@ -6,8 +6,10 @@ import type { SessionResponse } from "./types.js"; /** * Create a new chat session + * @param currentNoteId - Optional current note ID for context + * @returns The noteId of the created chat note */ -export async function createChatSession(currentNoteId?: string): Promise<{chatNoteId: string | null, noteId: string | null}> { +export async function createChatSession(currentNoteId?: string): Promise { try { const resp = await server.post('llm/chat', { title: 'Note Chat', @@ -15,48 +17,42 @@ export async function createChatSession(currentNoteId?: string): Promise<{chatNo }); if (resp && resp.id) { - // The backend might provide the noteId separately from the chatNoteId - // If noteId is provided, use it; otherwise, we'll need to query for it separately - return { - chatNoteId: resp.id, - noteId: resp.noteId || null - }; + // Backend returns the chat note ID as 'id' + return resp.id; } } catch (error) { console.error('Failed to create chat session:', error); } - return { - chatNoteId: null, - noteId: null - }; + return null; } /** - * Check if a session exists + * Check if a chat note exists + * @param noteId - The ID of the chat note */ -export async function checkSessionExists(chatNoteId: string): Promise { +export async function checkSessionExists(noteId: string): Promise { try { - // Validate that we have a proper note ID format, not a session ID - // Note IDs in Trilium are typically longer or in a different format - if (chatNoteId && chatNoteId.length === 16 && /^[A-Za-z0-9]+$/.test(chatNoteId)) { - console.warn(`Invalid note ID format detected: ${chatNoteId} appears to be a legacy session ID`); - return false; - } - - const sessionCheck = await server.getWithSilentNotFound(`llm/chat/${chatNoteId}`); + const sessionCheck = await server.getWithSilentNotFound(`llm/chat/${noteId}`); return !!(sessionCheck && sessionCheck.id); } catch (error: any) { - console.log(`Error checking chat note ${chatNoteId}:`, error); + console.log(`Error checking chat note ${noteId}:`, error); return false; } } /** * Set up streaming response via WebSocket + * @param noteId - The ID of the chat note + * @param messageParams - Message parameters + * @param onContentUpdate - Callback for content updates + * @param onThinkingUpdate - Callback for thinking updates + * @param onToolExecution - Callback for tool execution + * @param onComplete - Callback for completion + * @param onError - Callback for errors */ export async function setupStreamingResponse( - chatNoteId: string, + noteId: string, messageParams: any, onContentUpdate: (content: string, isDone?: boolean) => void, onThinkingUpdate: (thinking: string) => void, @@ -64,13 +60,6 @@ export async function setupStreamingResponse( onComplete: () => void, onError: (error: Error) => void ): Promise { - // Validate that we have a proper note ID format, not a session ID - if (chatNoteId && chatNoteId.length === 16 && /^[A-Za-z0-9]+$/.test(chatNoteId)) { - console.error(`Invalid note ID format: ${chatNoteId} appears to be a legacy session ID`); - onError(new Error("Invalid note ID format - using a legacy session ID")); - return; - } - return new Promise((resolve, reject) => { let assistantResponse = ''; let postToolResponse = ''; // Separate accumulator for post-tool execution content @@ -87,12 +76,12 @@ export async function setupStreamingResponse( // Create a unique identifier for this response process const responseId = `llm-stream-${Date.now()}-${Math.floor(Math.random() * 1000)}`; - console.log(`[${responseId}] Setting up WebSocket streaming for chat note ${chatNoteId}`); + console.log(`[${responseId}] Setting up WebSocket streaming for chat note ${noteId}`); // Send the initial request to initiate streaming (async () => { try { - const streamResponse = await server.post(`llm/chat/${chatNoteId}/messages/stream`, { + const streamResponse = await server.post(`llm/chat/${noteId}/messages/stream`, { content: messageParams.content, useAdvancedContext: messageParams.useAdvancedContext, showThinking: messageParams.showThinking, @@ -158,7 +147,7 @@ export async function setupStreamingResponse( const message = customEvent.detail; // Only process messages for our chat note - if (!message || message.chatNoteId !== chatNoteId) { + if (!message || message.chatNoteId !== noteId) { return; } @@ -172,12 +161,12 @@ export async function setupStreamingResponse( cleanupTimeoutId = null; } - console.log(`[${responseId}] LLM Stream message received via CustomEvent: chatNoteId=${chatNoteId}, content=${!!message.content}, contentLength=${message.content?.length || 0}, thinking=${!!message.thinking}, toolExecution=${!!message.toolExecution}, done=${!!message.done}, type=${message.type || 'llm-stream'}`); + console.log(`[${responseId}] LLM Stream message received via CustomEvent: chatNoteId=${noteId}, content=${!!message.content}, contentLength=${message.content?.length || 0}, thinking=${!!message.thinking}, toolExecution=${!!message.toolExecution}, done=${!!message.done}, type=${message.type || 'llm-stream'}`); // Mark first message received if (!receivedAnyMessage) { receivedAnyMessage = true; - console.log(`[${responseId}] First message received for chat note ${chatNoteId}`); + console.log(`[${responseId}] First message received for chat note ${noteId}`); // Clear the initial timeout since we've received a message if (initialTimeoutId !== null) { @@ -298,7 +287,7 @@ export async function setupStreamingResponse( // Set new timeout timeoutId = window.setTimeout(() => { - console.warn(`[${responseId}] Stream timeout for chat note ${chatNoteId}`); + console.warn(`[${responseId}] Stream timeout for chat note ${noteId}`); // Clean up performCleanup(); @@ -369,7 +358,7 @@ export async function setupStreamingResponse( // Handle completion if (message.done) { - console.log(`[${responseId}] Stream completed for chat note ${chatNoteId}, has content: ${!!message.content}, content length: ${message.content?.length || 0}, current response: ${assistantResponse.length} chars`); + console.log(`[${responseId}] Stream completed for chat note ${noteId}, has content: ${!!message.content}, content length: ${message.content?.length || 0}, current response: ${assistantResponse.length} chars`); // Dump message content to console for debugging if (message.content) { @@ -428,9 +417,9 @@ export async function setupStreamingResponse( // Set initial timeout for receiving any message initialTimeoutId = window.setTimeout(() => { - console.warn(`[${responseId}] No messages received for initial period in chat note ${chatNoteId}`); + console.warn(`[${responseId}] No messages received for initial period in chat note ${noteId}`); if (!receivedAnyMessage) { - console.error(`[${responseId}] WebSocket connection not established for chat note ${chatNoteId}`); + console.error(`[${responseId}] WebSocket connection not established for chat note ${noteId}`); if (timeoutId !== null) { window.clearTimeout(timeoutId); @@ -463,15 +452,9 @@ function cleanupEventListener(listener: ((event: Event) => void) | null): void { /** * Get a direct response from the server without streaming */ -export async function getDirectResponse(chatNoteId: string, messageParams: any): Promise { +export async function getDirectResponse(noteId: string, messageParams: any): Promise { try { - // Validate that we have a proper note ID format, not a session ID - if (chatNoteId && chatNoteId.length === 16 && /^[A-Za-z0-9]+$/.test(chatNoteId)) { - console.error(`Invalid note ID format: ${chatNoteId} appears to be a legacy session ID`); - throw new Error("Invalid note ID format - using a legacy session ID"); - } - - const postResponse = await server.post(`llm/chat/${chatNoteId}/messages`, { + const postResponse = await server.post(`llm/chat/${noteId}/messages`, { message: messageParams.content, includeContext: messageParams.useAdvancedContext, options: { diff --git a/apps/client/src/widgets/llm_chat/llm_chat_panel.ts b/apps/client/src/widgets/llm_chat/llm_chat_panel.ts index 32ffab50d..57a95fd5b 100644 --- a/apps/client/src/widgets/llm_chat/llm_chat_panel.ts +++ b/apps/client/src/widgets/llm_chat/llm_chat_panel.ts @@ -37,9 +37,10 @@ export default class LlmChatPanel extends BasicWidget { private thinkingBubble!: HTMLElement; private thinkingText!: HTMLElement; private thinkingToggle!: HTMLElement; - private chatNoteId: string | null = null; - private noteId: string | null = null; // The actual noteId for the Chat Note - private currentNoteId: string | null = null; + + // Simplified to just use noteId - this represents the AI Chat note we're working with + private noteId: string | null = null; + private currentNoteId: string | null = null; // The note providing context (for regular notes) private _messageHandlerId: number | null = null; private _messageHandler: any = null; @@ -90,12 +91,21 @@ export default class LlmChatPanel extends BasicWidget { this.messages = messages; } - public getChatNoteId(): string | null { - return this.chatNoteId; + public getNoteId(): string | null { + return this.noteId; } - public setChatNoteId(chatNoteId: string | null): void { - this.chatNoteId = chatNoteId; + public setNoteId(noteId: string | null): void { + this.noteId = noteId; + } + + // Deprecated - keeping for backward compatibility but mapping to noteId + public getChatNoteId(): string | null { + return this.noteId; + } + + public setChatNoteId(noteId: string | null): void { + this.noteId = noteId; } public getNoteContextChatMessages(): HTMLElement { @@ -307,10 +317,16 @@ export default class LlmChatPanel extends BasicWidget { } } - const dataToSave: ChatData = { + // Only save if we have a valid note ID + if (!this.noteId) { + console.warn('Cannot save chat data: no noteId available'); + return; + } + + const dataToSave = { messages: this.messages, - chatNoteId: this.chatNoteId, noteId: this.noteId, + chatNoteId: this.noteId, // For backward compatibility toolSteps: toolSteps, // Add sources if we have them sources: this.sources || [], @@ -325,7 +341,7 @@ export default class LlmChatPanel extends BasicWidget { } }; - console.log(`Saving chat data with chatNoteId: ${this.chatNoteId}, noteId: ${this.noteId}, ${toolSteps.length} tool steps, ${this.sources?.length || 0} sources, ${toolExecutions.length} tool executions`); + console.log(`Saving chat data with noteId: ${this.noteId}, ${toolSteps.length} tool steps, ${this.sources?.length || 0} sources, ${toolExecutions.length} tool executions`); // Save the data to the note attribute via the callback // This is the ONLY place we should save data, letting the container widget handle persistence @@ -400,7 +416,6 @@ export default class LlmChatPanel extends BasicWidget { // Load Chat Note ID if available if (savedData.noteId) { console.log(`Using noteId as Chat Note ID: ${savedData.noteId}`); - this.chatNoteId = savedData.noteId; this.noteId = savedData.noteId; } else { console.log(`No noteId found in saved data, cannot load chat session`); @@ -550,6 +565,15 @@ export default class LlmChatPanel extends BasicWidget { // Get current note context if needed const currentActiveNoteId = appContext.tabManager.getActiveContext()?.note?.noteId || null; + // For AI Chat notes, the note itself IS the chat session + // So currentNoteId and noteId should be the same + if (this.noteId && currentActiveNoteId === this.noteId) { + // We're in an AI Chat note - don't reset, just load saved data + console.log(`Refreshing AI Chat note ${this.noteId} - loading saved data`); + await this.loadSavedData(); + return; + } + // If we're switching to a different note, we need to reset if (this.currentNoteId !== currentActiveNoteId) { console.log(`Note ID changed from ${this.currentNoteId} to ${currentActiveNoteId}, resetting chat panel`); @@ -557,7 +581,6 @@ export default class LlmChatPanel extends BasicWidget { // Reset the UI and data this.noteContextChatMessages.innerHTML = ''; this.messages = []; - this.chatNoteId = null; this.noteId = null; // Also reset the chat note ID this.hideSources(); // Hide any sources from previous note @@ -569,7 +592,7 @@ export default class LlmChatPanel extends BasicWidget { const hasSavedData = await this.loadSavedData(); // Only create a new session if we don't have a session or saved data - if (!this.chatNoteId || !this.noteId || !hasSavedData) { + if (!this.noteId || !hasSavedData) { // Create a new chat session await this.createChatSession(); } @@ -580,19 +603,15 @@ export default class LlmChatPanel extends BasicWidget { */ private async createChatSession() { try { - // Create a new chat session, passing the current note ID if it exists - const { chatNoteId, noteId } = await createChatSession( - this.currentNoteId ? this.currentNoteId : undefined - ); + // If we already have a noteId (for AI Chat notes), use it + const contextNoteId = this.noteId || this.currentNoteId; - if (chatNoteId) { - // If we got back an ID from the API, use it - this.chatNoteId = chatNoteId; - - // For new sessions, the noteId should equal the chatNoteId - // This ensures we're using the note ID consistently - this.noteId = noteId || chatNoteId; + // Create a new chat session, passing the context note ID + const noteId = await createChatSession(contextNoteId ? contextNoteId : undefined); + if (noteId) { + // Set the note ID for this chat + this.noteId = noteId; console.log(`Created new chat session with noteId: ${this.noteId}`); } else { throw new Error("Failed to create chat session - no ID returned"); @@ -645,7 +664,7 @@ export default class LlmChatPanel extends BasicWidget { const showThinking = this.showThinkingCheckbox.checked; // Add logging to verify parameters - console.log(`Sending message with: useAdvancedContext=${useAdvancedContext}, showThinking=${showThinking}, noteId=${this.currentNoteId}, sessionId=${this.chatNoteId}`); + console.log(`Sending message with: useAdvancedContext=${useAdvancedContext}, showThinking=${showThinking}, noteId=${this.currentNoteId}, sessionId=${this.noteId}`); // Create the message parameters const messageParams = { @@ -695,11 +714,11 @@ export default class LlmChatPanel extends BasicWidget { await validateEmbeddingProviders(this.validationWarning); // Make sure we have a valid session - if (!this.chatNoteId) { + if (!this.noteId) { // If no session ID, create a new session await this.createChatSession(); - if (!this.chatNoteId) { + if (!this.noteId) { // If still no session ID, show error and return console.error("Failed to create chat session"); toastService.showError("Failed to create chat session"); @@ -730,7 +749,7 @@ export default class LlmChatPanel extends BasicWidget { await this.saveCurrentData(); // Add logging to verify parameters - console.log(`Sending message with: useAdvancedContext=${useAdvancedContext}, showThinking=${showThinking}, noteId=${this.currentNoteId}, sessionId=${this.chatNoteId}`); + console.log(`Sending message with: useAdvancedContext=${useAdvancedContext}, showThinking=${showThinking}, noteId=${this.currentNoteId}, sessionId=${this.noteId}`); // Create the message parameters const messageParams = { @@ -767,12 +786,12 @@ export default class LlmChatPanel extends BasicWidget { */ private async handleDirectResponse(messageParams: any): Promise { try { - if (!this.chatNoteId) return false; + if (!this.noteId) return false; - console.log(`Getting direct response using sessionId: ${this.chatNoteId} (noteId: ${this.noteId})`); + console.log(`Getting direct response using sessionId: ${this.noteId} (noteId: ${this.noteId})`); // Get a direct response from the server - const postResponse = await getDirectResponse(this.chatNoteId, messageParams); + const postResponse = await getDirectResponse(this.noteId, messageParams); // If the POST request returned content directly, display it if (postResponse && postResponse.content) { @@ -845,11 +864,11 @@ export default class LlmChatPanel extends BasicWidget { * Set up streaming response via WebSocket */ private async setupStreamingResponse(messageParams: any): Promise { - if (!this.chatNoteId) { + if (!this.noteId) { throw new Error("No session ID available"); } - console.log(`Setting up streaming response using sessionId: ${this.chatNoteId} (noteId: ${this.noteId})`); + console.log(`Setting up streaming response using sessionId: ${this.noteId} (noteId: ${this.noteId})`); // Store tool executions captured during streaming const toolExecutionsCache: Array<{ @@ -862,7 +881,7 @@ export default class LlmChatPanel extends BasicWidget { }> = []; return setupStreamingResponse( - this.chatNoteId, + this.noteId, messageParams, // Content update handler (content: string, isDone: boolean = false) => { @@ -898,7 +917,7 @@ export default class LlmChatPanel extends BasicWidget { similarity?: number; content?: string; }>; - }>(`llm/chat/${this.chatNoteId}`) + }>(`llm/chat/${this.noteId}`) .then((sessionData) => { console.log("Got updated session data:", sessionData); diff --git a/apps/client/src/widgets/llm_chat/types.ts b/apps/client/src/widgets/llm_chat/types.ts index 300a7856a..7181651d0 100644 --- a/apps/client/src/widgets/llm_chat/types.ts +++ b/apps/client/src/widgets/llm_chat/types.ts @@ -11,7 +11,7 @@ export interface ChatResponse { export interface SessionResponse { id: string; title: string; - noteId?: string; + noteId: string; // The ID of the chat note } export interface ToolExecutionStep { @@ -33,8 +33,8 @@ export interface MessageData { export interface ChatData { messages: MessageData[]; - chatNoteId: string | null; - noteId?: string | null; + noteId: string; // The ID of the chat note + chatNoteId?: string; // Deprecated - kept for backward compatibility, should equal noteId toolSteps: ToolExecutionStep[]; sources?: Array<{ noteId: string; diff --git a/apps/client/src/widgets/type_widgets/ai_chat.ts b/apps/client/src/widgets/type_widgets/ai_chat.ts index e96cf5f20..f733b499b 100644 --- a/apps/client/src/widgets/type_widgets/ai_chat.ts +++ b/apps/client/src/widgets/type_widgets/ai_chat.ts @@ -94,6 +94,11 @@ export default class AiChatTypeWidget extends TypeWidget { this.llmChatPanel.clearNoteContextChatMessages(); this.llmChatPanel.setMessages([]); + // Set the note ID for the chat panel + if (note) { + this.llmChatPanel.setNoteId(note.noteId); + } + // This will load saved data via the getData callback await this.llmChatPanel.refresh(); this.isInitialized = true; @@ -130,7 +135,7 @@ export default class AiChatTypeWidget extends TypeWidget { // Reset the chat panel UI this.llmChatPanel.clearNoteContextChatMessages(); this.llmChatPanel.setMessages([]); - this.llmChatPanel.setChatNoteId(null); + this.llmChatPanel.setNoteId(this.note.noteId); } // Call the parent method to refresh @@ -152,6 +157,7 @@ export default class AiChatTypeWidget extends TypeWidget { // Make sure the chat panel has the current note ID if (this.note) { this.llmChatPanel.setCurrentNoteId(this.note.noteId); + this.llmChatPanel.setNoteId(this.note.noteId); } this.initPromise = (async () => { @@ -186,7 +192,7 @@ export default class AiChatTypeWidget extends TypeWidget { // Format the data properly - this is the canonical format of the data const formattedData = { messages: data.messages || [], - chatNoteId: data.chatNoteId || this.note.noteId, + noteId: this.note.noteId, // Always use the note's own ID toolSteps: data.toolSteps || [], sources: data.sources || [], metadata: { diff --git a/apps/server/src/routes/api/llm.ts b/apps/server/src/routes/api/llm.ts index 013ce779d..be046bdd6 100644 --- a/apps/server/src/routes/api/llm.ts +++ b/apps/server/src/routes/api/llm.ts @@ -814,10 +814,11 @@ async function streamMessage(req: Request, res: Response) { throw new Error('Content cannot be empty'); } - // Check if session exists - const session = restChatService.getSessions().get(chatNoteId); + // Get or create session from Chat Note + // This will check the sessions store first, and if not found, create from the Chat Note + const session = await restChatService.getOrCreateSessionFromChatNote(chatNoteId, true); if (!session) { - throw new Error('Chat not found'); + throw new Error('Chat not found and could not be created from note'); } // Update last active timestamp diff --git a/apps/server/src/services/llm/chat/rest_chat_service.ts b/apps/server/src/services/llm/chat/rest_chat_service.ts index 0a400ad91..432cc6e08 100644 --- a/apps/server/src/services/llm/chat/rest_chat_service.ts +++ b/apps/server/src/services/llm/chat/rest_chat_service.ts @@ -480,27 +480,56 @@ class RestChatService { const options: any = req.body || {}; const title = options.title || 'Chat Session'; - // Use the currentNoteId as the chatNoteId if provided - let chatNoteId = options.chatNoteId; + // Determine the note ID for the chat + let noteId = options.noteId || options.chatNoteId; // Accept either name for backward compatibility - // If currentNoteId is provided but chatNoteId is not, use currentNoteId - if (!chatNoteId && options.currentNoteId) { - chatNoteId = options.currentNoteId; - log.info(`Using provided currentNoteId ${chatNoteId} as chatNoteId`); + // If currentNoteId is provided, check if it's already an AI Chat note + if (!noteId && options.currentNoteId) { + // Import becca to check note type + const becca = (await import('../../../becca/becca.js')).default; + const note = becca.notes[options.currentNoteId]; + + // Check if this is an AI Chat note by looking at its content structure + if (note) { + try { + const content = note.getContent(); + if (content) { + const contentStr = typeof content === 'string' ? content : content.toString(); + const parsedContent = JSON.parse(contentStr); + // AI Chat notes have a messages array and noteId in their content + if (parsedContent.messages && Array.isArray(parsedContent.messages) && parsedContent.noteId) { + // This looks like an AI Chat note - use it directly + noteId = options.currentNoteId; + log.info(`Using existing AI Chat note ${noteId} as session`); + } + } + } catch (e) { + // Not JSON content, so not an AI Chat note + } + } + + if (!noteId) { + log.info(`Creating new chat note from context of note ${options.currentNoteId}`); + // Don't use the currentNoteId as the chat note ID - create a new one + } } - // If we still don't have a chatNoteId, create a new Chat Note - if (!chatNoteId) { + // If we don't have a noteId, create a new Chat Note + if (!noteId) { // Create a new Chat Note via the storage service const chatStorageService = (await import('../../llm/chat_storage_service.js')).default; const newChat = await chatStorageService.createChat(title); - chatNoteId = newChat.id; - log.info(`Created new Chat Note with ID: ${chatNoteId}`); + noteId = newChat.id; + log.info(`Created new Chat Note with ID: ${noteId}`); + } else { + // We have a noteId - this means we're working with an existing aiChat note + // Don't create another note, just use the existing one + log.info(`Using existing Chat Note with ID: ${noteId}`); } - // Create a new session through our session store + // Create a new session through our session store using the note ID const session = SessionsStore.createSession({ - chatNoteId, + chatNoteId: noteId, // This is really the noteId of the chat note title, systemPrompt: options.systemPrompt, contextNoteId: options.contextNoteId, @@ -511,10 +540,10 @@ class RestChatService { }); return { - id: session.id, + id: session.id, // This will be the same as noteId title: session.title, createdAt: session.createdAt, - noteId: chatNoteId // Return the note ID explicitly + noteId: noteId // Return the note ID for clarity }; } catch (error: any) { log.error(`Error creating LLM session: ${error.message || 'Unknown error'}`); From f6af617f6bfa3980e06eb614141afe4950821635 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Mon, 2 Jun 2025 02:38:21 +0000 Subject: [PATCH 02/18] feat(llm): redo chat storage, part 2 --- .../src/widgets/llm_chat/llm_chat_panel.ts | 3 +- .../services/llm/chat/rest_chat_service.ts | 56 ++++++++++++++++++- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/apps/client/src/widgets/llm_chat/llm_chat_panel.ts b/apps/client/src/widgets/llm_chat/llm_chat_panel.ts index 57a95fd5b..fc6a28f26 100644 --- a/apps/client/src/widgets/llm_chat/llm_chat_panel.ts +++ b/apps/client/src/widgets/llm_chat/llm_chat_panel.ts @@ -69,7 +69,6 @@ export default class LlmChatPanel extends BasicWidget { totalTokens?: number; }; } = { - model: 'default', temperature: 0.7, toolExecutions: [] }; @@ -332,7 +331,7 @@ export default class LlmChatPanel extends BasicWidget { sources: this.sources || [], // Add metadata metadata: { - model: this.metadata?.model || 'default', + model: this.metadata?.model || undefined, provider: this.metadata?.provider || undefined, temperature: this.metadata?.temperature || 0.7, lastUpdated: new Date().toISOString(), diff --git a/apps/server/src/services/llm/chat/rest_chat_service.ts b/apps/server/src/services/llm/chat/rest_chat_service.ts index 432cc6e08..eb57276bd 100644 --- a/apps/server/src/services/llm/chat/rest_chat_service.ts +++ b/apps/server/src/services/llm/chat/rest_chat_service.ts @@ -267,7 +267,8 @@ class RestChatService { systemPrompt: session?.messages.find(m => m.role === 'system')?.content, temperature: session?.metadata.temperature, maxTokens: session?.metadata.maxTokens, - model: session?.metadata.model, + // Get the user's preferred model if session model is 'default' or not set + model: await this.getPreferredModel(session?.metadata.model), // Set stream based on request type, but ensure it's explicitly a boolean value // GET requests or format=stream parameter indicates streaming should be used stream: !!(req.method === 'GET' || req.query.format === 'stream' || req.query.stream === 'true'), @@ -702,6 +703,59 @@ class RestChatService { // Create from Chat Note return await this.createSessionFromChatNote(noteId); } + + /** + * Get the user's preferred model + */ + async getPreferredModel(sessionModel?: string): Promise { + // If the session already has a valid model (not 'default'), use it + if (sessionModel && sessionModel !== 'default') { + return sessionModel; + } + + try { + // Get provider precedence list (same logic as model selection stage) + const providerPrecedence = await options.getOption('aiProviderPrecedence'); + let defaultProvider = 'openai'; + let defaultModelName = 'gpt-3.5-turbo'; + + if (providerPrecedence) { + // Parse provider precedence list + let providers: string[] = []; + if (providerPrecedence.includes(',')) { + providers = providerPrecedence.split(',').map(p => p.trim()); + } else if (providerPrecedence.startsWith('[') && providerPrecedence.endsWith(']')) { + providers = JSON.parse(providerPrecedence); + } else { + providers = [providerPrecedence]; + } + + // Get first available provider + if (providers.length > 0) { + const firstProvider = providers[0]; + defaultProvider = firstProvider; + + // Get provider-specific default model + if (firstProvider === 'openai') { + const model = await options.getOption('openaiDefaultModel'); + if (model) defaultModelName = model; + } else if (firstProvider === 'anthropic') { + const model = await options.getOption('anthropicDefaultModel'); + if (model) defaultModelName = model; + } else if (firstProvider === 'ollama') { + const model = await options.getOption('ollamaDefaultModel'); + if (model) defaultModelName = model; + } + } + } + + log.info(`Selected user's preferred model: ${defaultModelName} from provider: ${defaultProvider}`); + return defaultModelName; + } catch (error) { + log.error(`Error getting user's preferred model: ${error}`); + return 'gpt-3.5-turbo'; // Fallback + } + } } // Create singleton instance From dcab4caee3de567c48a85e28a6e15bdae51b30d9 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Mon, 2 Jun 2025 15:12:08 +0000 Subject: [PATCH 03/18] feat(llm): redo chat storage, part 3 --- .../src/services/llm/ai_service_manager.ts | 31 +++++++++++++------ .../stages/message_preparation_stage.ts | 20 ++++++------ .../pipeline/stages/model_selection_stage.ts | 3 +- 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/apps/server/src/services/llm/ai_service_manager.ts b/apps/server/src/services/llm/ai_service_manager.ts index fbbc12cb5..e76c13ae2 100644 --- a/apps/server/src/services/llm/ai_service_manager.ts +++ b/apps/server/src/services/llm/ai_service_manager.ts @@ -155,7 +155,7 @@ export class AIServiceManager implements IAIServiceManager { // Get precedence list from options let precedenceList: string[] = ['openai']; // Default to openai if not set const precedenceOption = await options.getOption('aiProviderPrecedence'); - + if (precedenceOption) { try { if (precedenceOption.startsWith('[') && precedenceOption.endsWith(']')) { @@ -171,10 +171,10 @@ export class AIServiceManager implements IAIServiceManager { log.error(`Error parsing precedence list: ${e}`); } } - + // Check for configuration issues with providers in the precedence list const configIssues: string[] = []; - + // Check each provider in the precedence list for proper configuration for (const provider of precedenceList) { if (provider === 'openai') { @@ -198,20 +198,20 @@ export class AIServiceManager implements IAIServiceManager { } // Add checks for other providers as needed } - + // Return warning message if there are configuration issues if (configIssues.length > 0) { let message = 'There are issues with your AI provider configuration:'; - + for (const issue of configIssues) { message += `\n• ${issue}`; } - + message += '\n\nPlease check your AI settings.'; - + // Log warning to console log.error('AI Provider Configuration Warning: ' + message); - + return message; } @@ -279,9 +279,19 @@ export class AIServiceManager implements IAIServiceManager { // If a specific provider is requested and available, use it if (options.model && options.model.includes(':')) { - const [providerName, modelName] = options.model.split(':'); + // Check if this is a provider prefix (e.g., "ollama:qwen3:30b") + // vs a model name with version (e.g., "qwen3:30b") + const parts = options.model.split(':'); + + // Only treat as provider:model if the first part is a known provider + const knownProviders = ['openai', 'anthropic', 'ollama', 'local']; + const potentialProvider = parts[0]; + + if (knownProviders.includes(potentialProvider) && availableProviders.includes(potentialProvider as ServiceProviders)) { + // This is a provider:model format + const providerName = potentialProvider; + const modelName = parts.slice(1).join(':'); // Rejoin the rest as model name - if (availableProviders.includes(providerName as ServiceProviders)) { try { const modifiedOptions = { ...options, model: modelName }; log.info(`[AIServiceManager] Using provider ${providerName} from model prefix with modifiedOptions.stream: ${modifiedOptions.stream}`); @@ -291,6 +301,7 @@ export class AIServiceManager implements IAIServiceManager { // If the specified provider fails, continue with the fallback providers } } + // If not a provider prefix, treat the entire string as a model name and continue with normal provider selection } // Try each provider in order until one succeeds diff --git a/apps/server/src/services/llm/pipeline/stages/message_preparation_stage.ts b/apps/server/src/services/llm/pipeline/stages/message_preparation_stage.ts index 753bc6a28..7f129b26d 100644 --- a/apps/server/src/services/llm/pipeline/stages/message_preparation_stage.ts +++ b/apps/server/src/services/llm/pipeline/stages/message_preparation_stage.ts @@ -20,44 +20,44 @@ export class MessagePreparationStage extends BasePipelineStage { const { messages, context, systemPrompt, options } = input; - + // Determine provider from model string if available (format: "provider:model") let provider = 'default'; if (options?.model && options.model.includes(':')) { const [providerName] = options.model.split(':'); provider = providerName; } - + // Check if tools are enabled const toolsEnabled = options?.enableTools === true; - + log.info(`Preparing messages for provider: ${provider}, context: ${!!context}, system prompt: ${!!systemPrompt}, tools: ${toolsEnabled}`); - + // Get appropriate formatter for this provider const formatter = MessageFormatterFactory.getFormatter(provider); - + // Determine the system prompt to use let finalSystemPrompt = systemPrompt || SYSTEM_PROMPTS.DEFAULT_SYSTEM_PROMPT; - + // If tools are enabled, enhance system prompt with tools guidance if (toolsEnabled) { const toolCount = toolRegistry.getAllTools().length; const toolsPrompt = `You have access to ${toolCount} tools to help you respond. When you need information that might be in the user's notes, use the search_notes tool to find relevant content or the read_note tool to read a specific note by ID. Use tools when specific information is required rather than making assumptions.`; - + // Add tools guidance to system prompt finalSystemPrompt = finalSystemPrompt + '\n\n' + toolsPrompt; log.info(`Enhanced system prompt with tools guidance: ${toolCount} tools available`); } - + // Format messages using provider-specific approach const formattedMessages = formatter.formatMessages( messages, finalSystemPrompt, context ); - + log.info(`Formatted ${messages.length} messages into ${formattedMessages.length} messages for provider: ${provider}`); - + return { messages: formattedMessages }; } } diff --git a/apps/server/src/services/llm/pipeline/stages/model_selection_stage.ts b/apps/server/src/services/llm/pipeline/stages/model_selection_stage.ts index e5406997d..0830b0bb8 100644 --- a/apps/server/src/services/llm/pipeline/stages/model_selection_stage.ts +++ b/apps/server/src/services/llm/pipeline/stages/model_selection_stage.ts @@ -234,7 +234,8 @@ export class ModelSelectionStage extends BasePipelineStage Date: Mon, 2 Jun 2025 21:36:19 +0000 Subject: [PATCH 04/18] refactor(llm): integrate new configuration system for provider management and model selection --- .../src/services/llm/ai_service_manager.ts | 165 +++----- .../services/llm/chat/rest_chat_service.ts | 4 +- .../llm/config/configuration_helpers.ts | 130 ++++++ .../llm/config/configuration_manager.ts | 373 ++++++++++++++++++ .../interfaces/configuration_interfaces.ts | 108 +++++ .../pipeline/stages/model_selection_stage.ts | 156 +++----- 6 files changed, 730 insertions(+), 206 deletions(-) create mode 100644 apps/server/src/services/llm/config/configuration_helpers.ts create mode 100644 apps/server/src/services/llm/config/configuration_manager.ts create mode 100644 apps/server/src/services/llm/interfaces/configuration_interfaces.ts diff --git a/apps/server/src/services/llm/ai_service_manager.ts b/apps/server/src/services/llm/ai_service_manager.ts index e76c13ae2..b8ec8325e 100644 --- a/apps/server/src/services/llm/ai_service_manager.ts +++ b/apps/server/src/services/llm/ai_service_manager.ts @@ -18,6 +18,18 @@ import type { } from './interfaces/ai_service_interfaces.js'; import type { NoteSearchResult } from './interfaces/context_interfaces.js'; +// Import new configuration system +import { + getProviderPrecedence, + getPreferredProvider, + getEmbeddingProviderPrecedence, + parseModelIdentifier, + isAIEnabled, + getDefaultModelForProvider, + clearConfigurationCache +} from './config/configuration_helpers.js'; +import type { ProviderType } from './interfaces/configuration_interfaces.js'; + /** * Interface representing relevant note context */ @@ -71,7 +83,7 @@ export class AIServiceManager implements IAIServiceManager { } /** - * Update the provider precedence order from saved options + * Update the provider precedence order using the new configuration system * Returns true if successful, false if options not available yet */ updateProviderOrder(): boolean { @@ -80,54 +92,17 @@ export class AIServiceManager implements IAIServiceManager { } try { - // Default precedence: openai, anthropic, ollama - const defaultOrder: ServiceProviders[] = ['openai', 'anthropic', 'ollama']; - - // Get custom order from options - const customOrder = options.getOption('aiProviderPrecedence'); - - if (customOrder) { - try { - // Try to parse as JSON first - let parsed; - - // Handle both array in JSON format and simple string format - if (customOrder.startsWith('[') && customOrder.endsWith(']')) { - parsed = JSON.parse(customOrder); - } else if (typeof customOrder === 'string') { - // If it's a string with commas, split it - if (customOrder.includes(',')) { - parsed = customOrder.split(',').map(p => p.trim()); - } else { - // If it's a simple string (like "ollama"), convert to single-item array - parsed = [customOrder]; - } - } else { - // Fallback to default - parsed = defaultOrder; - } - - // Validate that all providers are valid - if (Array.isArray(parsed) && - parsed.every(p => Object.keys(this.services).includes(p))) { - this.providerOrder = parsed as ServiceProviders[]; - } else { - log.info('Invalid AI provider precedence format, using defaults'); - this.providerOrder = defaultOrder; - } - } catch (e) { - log.error(`Failed to parse AI provider precedence: ${e}`); - this.providerOrder = defaultOrder; - } - } else { - this.providerOrder = defaultOrder; - } + // Use async helper but handle it synchronously for now + // In a real refactor, this method should become async + getProviderPrecedence().then(providers => { + this.providerOrder = providers as ServiceProviders[]; + log.info(`Updated provider order: ${providers.join(', ')}`); + }).catch(error => { + log.error(`Failed to get provider precedence: ${error}`); + // Keep default order + }); this.initialized = true; - - // Remove the validateEmbeddingProviders call since we now do validation on the client - // this.validateEmbeddingProviders(); - return true; } catch (error) { // If options table doesn't exist yet, use defaults @@ -138,39 +113,18 @@ export class AIServiceManager implements IAIServiceManager { } /** - * Validate embedding providers configuration - * - Check if embedding default provider is in provider precedence list - * - Check if all providers in precedence list and default provider are enabled - * - * @returns A warning message if there are issues, or null if everything is fine + * Validate embedding providers configuration using the new configuration system */ async validateEmbeddingProviders(): Promise { try { - // Check if AI is enabled, if not, skip validation - const aiEnabled = await options.getOptionBool('aiEnabled'); + // Check if AI is enabled using the new helper + const aiEnabled = await isAIEnabled(); if (!aiEnabled) { return null; } - // Get precedence list from options - let precedenceList: string[] = ['openai']; // Default to openai if not set - const precedenceOption = await options.getOption('aiProviderPrecedence'); - - if (precedenceOption) { - try { - if (precedenceOption.startsWith('[') && precedenceOption.endsWith(']')) { - precedenceList = JSON.parse(precedenceOption); - } else if (typeof precedenceOption === 'string') { - if (precedenceOption.includes(',')) { - precedenceList = precedenceOption.split(',').map(p => p.trim()); - } else { - precedenceList = [precedenceOption]; - } - } - } catch (e) { - log.error(`Error parsing precedence list: ${e}`); - } - } + // Get precedence list using the new helper (no string parsing!) + const precedenceList = await getEmbeddingProviderPrecedence(); // Check for configuration issues with providers in the precedence list const configIssues: string[] = []; @@ -279,25 +233,16 @@ export class AIServiceManager implements IAIServiceManager { // If a specific provider is requested and available, use it if (options.model && options.model.includes(':')) { - // Check if this is a provider prefix (e.g., "ollama:qwen3:30b") - // vs a model name with version (e.g., "qwen3:30b") - const parts = options.model.split(':'); - - // Only treat as provider:model if the first part is a known provider - const knownProviders = ['openai', 'anthropic', 'ollama', 'local']; - const potentialProvider = parts[0]; - - if (knownProviders.includes(potentialProvider) && availableProviders.includes(potentialProvider as ServiceProviders)) { - // This is a provider:model format - const providerName = potentialProvider; - const modelName = parts.slice(1).join(':'); // Rejoin the rest as model name + // Use the new configuration system to parse model identifier + const modelIdentifier = parseModelIdentifier(options.model); + if (modelIdentifier.provider && availableProviders.includes(modelIdentifier.provider as ServiceProviders)) { try { - const modifiedOptions = { ...options, model: modelName }; - log.info(`[AIServiceManager] Using provider ${providerName} from model prefix with modifiedOptions.stream: ${modifiedOptions.stream}`); - return await this.services[providerName as ServiceProviders].generateChatCompletion(messages, modifiedOptions); + const modifiedOptions = { ...options, model: modelIdentifier.modelId }; + log.info(`[AIServiceManager] Using provider ${modelIdentifier.provider} from model prefix with modifiedOptions.stream: ${modifiedOptions.stream}`); + return await this.services[modelIdentifier.provider as ServiceProviders].generateChatCompletion(messages, modifiedOptions); } catch (error) { - log.error(`Error with specified provider ${providerName}: ${error}`); + log.error(`Error with specified provider ${modelIdentifier.provider}: ${error}`); // If the specified provider fails, continue with the fallback providers } } @@ -401,9 +346,11 @@ export class AIServiceManager implements IAIServiceManager { } /** - * Get whether AI features are enabled from options + * Get whether AI features are enabled using the new configuration system */ getAIEnabled(): boolean { + // For synchronous compatibility, use the old method + // In a full refactor, this should be async return options.getOptionBool('aiEnabled'); } @@ -417,23 +364,7 @@ export class AIServiceManager implements IAIServiceManager { return; } - // Get provider precedence list - const precedenceOption = await options.getOption('embeddingProviderPrecedence'); - let precedenceList: string[] = []; - - if (precedenceOption) { - if (precedenceOption.startsWith('[') && precedenceOption.endsWith(']')) { - precedenceList = JSON.parse(precedenceOption); - } else if (typeof precedenceOption === 'string') { - if (precedenceOption.includes(',')) { - precedenceList = precedenceOption.split(',').map(p => p.trim()); - } else { - precedenceList = [precedenceOption]; - } - } - } - - // Check if we have enabled providers + // Use the new configuration system - no string parsing! const enabledProviders = await getEnabledEmbeddingProviders(); if (enabledProviders.length === 0) { @@ -456,10 +387,10 @@ export class AIServiceManager implements IAIServiceManager { try { log.info("Initializing AI service..."); - // Check if AI is enabled in options - const isAIEnabled = this.getAIEnabled(); + // Check if AI is enabled using the new helper + const isAIEnabled_value = await isAIEnabled(); - if (!isAIEnabled) { + if (!isAIEnabled_value) { log.info("AI features are disabled in options"); return; } @@ -597,7 +528,19 @@ export class AIServiceManager implements IAIServiceManager { } /** - * Get the preferred provider based on configuration + * Get the preferred provider based on configuration using the new system + */ + async getPreferredProviderAsync(): Promise { + try { + return await getPreferredProvider(); + } catch (error) { + log.error(`Error getting preferred provider: ${error}`); + return this.providerOrder[0]; + } + } + + /** + * Get the preferred provider based on configuration (sync version for compatibility) */ getPreferredProvider(): string { this.ensureInitialized(); diff --git a/apps/server/src/services/llm/chat/rest_chat_service.ts b/apps/server/src/services/llm/chat/rest_chat_service.ts index eb57276bd..4c1c9d2da 100644 --- a/apps/server/src/services/llm/chat/rest_chat_service.ts +++ b/apps/server/src/services/llm/chat/rest_chat_service.ts @@ -510,8 +510,8 @@ class RestChatService { } if (!noteId) { - log.info(`Creating new chat note from context of note ${options.currentNoteId}`); - // Don't use the currentNoteId as the chat note ID - create a new one + log.info(`Creating new chat note from context of note ${options.currentNoteId}`); + // Don't use the currentNoteId as the chat note ID - create a new one } } diff --git a/apps/server/src/services/llm/config/configuration_helpers.ts b/apps/server/src/services/llm/config/configuration_helpers.ts new file mode 100644 index 000000000..b4a9d5432 --- /dev/null +++ b/apps/server/src/services/llm/config/configuration_helpers.ts @@ -0,0 +1,130 @@ +import configurationManager from './configuration_manager.js'; +import type { + ProviderType, + ModelIdentifier, + ModelConfig, + ProviderPrecedenceConfig, + EmbeddingProviderPrecedenceConfig +} from '../interfaces/configuration_interfaces.js'; + +/** + * Helper functions for accessing AI configuration without string parsing + * Use these throughout the codebase instead of parsing strings directly + */ + +/** + * Get the ordered list of AI providers + */ +export async function getProviderPrecedence(): Promise { + const config = await configurationManager.getProviderPrecedence(); + return config.providers; +} + +/** + * Get the default/preferred AI provider + */ +export async function getPreferredProvider(): Promise { + const config = await configurationManager.getProviderPrecedence(); + return config.defaultProvider || config.providers[0]; +} + +/** + * Get the ordered list of embedding providers + */ +export async function getEmbeddingProviderPrecedence(): Promise { + const config = await configurationManager.getEmbeddingProviderPrecedence(); + return config.providers; +} + +/** + * Get the default embedding provider + */ +export async function getPreferredEmbeddingProvider(): Promise { + const config = await configurationManager.getEmbeddingProviderPrecedence(); + return config.defaultProvider || config.providers[0]; +} + +/** + * Parse a model identifier (handles "provider:model" format) + */ +export function parseModelIdentifier(modelString: string): ModelIdentifier { + return configurationManager.parseModelIdentifier(modelString); +} + +/** + * Create a model configuration from a model string + */ +export function createModelConfig(modelString: string, defaultProvider?: ProviderType): ModelConfig { + return configurationManager.createModelConfig(modelString, defaultProvider); +} + +/** + * Get the default model for a specific provider + */ +export async function getDefaultModelForProvider(provider: ProviderType): Promise { + const config = await configurationManager.getAIConfig(); + return config.defaultModels[provider]; +} + +/** + * Get provider settings for a specific provider + */ +export async function getProviderSettings(provider: ProviderType) { + const config = await configurationManager.getAIConfig(); + return config.providerSettings[provider]; +} + +/** + * Check if AI is enabled + */ +export async function isAIEnabled(): Promise { + const config = await configurationManager.getAIConfig(); + return config.enabled; +} + +/** + * Check if a provider has required configuration + */ +export async function isProviderConfigured(provider: ProviderType): Promise { + const settings = await getProviderSettings(provider); + + switch (provider) { + case 'openai': + return Boolean((settings as any)?.apiKey); + case 'anthropic': + return Boolean((settings as any)?.apiKey); + case 'ollama': + return Boolean((settings as any)?.baseUrl); + default: + return false; + } +} + +/** + * Get the first available (configured) provider from the precedence list + */ +export async function getFirstAvailableProvider(): Promise { + const providers = await getProviderPrecedence(); + + for (const provider of providers) { + if (await isProviderConfigured(provider)) { + return provider; + } + } + + return null; +} + +/** + * Validate the current AI configuration + */ +export async function validateConfiguration() { + return configurationManager.validateConfig(); +} + +/** + * Clear cached configuration (use when settings change) + */ +export function clearConfigurationCache(): void { + configurationManager.clearCache(); +} diff --git a/apps/server/src/services/llm/config/configuration_manager.ts b/apps/server/src/services/llm/config/configuration_manager.ts new file mode 100644 index 000000000..621e5d3f4 --- /dev/null +++ b/apps/server/src/services/llm/config/configuration_manager.ts @@ -0,0 +1,373 @@ +import options from '../../options.js'; +import log from '../../log.js'; +import type { + AIConfig, + ProviderPrecedenceConfig, + EmbeddingProviderPrecedenceConfig, + ModelIdentifier, + ModelConfig, + ProviderType, + EmbeddingProviderType, + ConfigValidationResult, + ProviderSettings, + OpenAISettings, + AnthropicSettings, + OllamaSettings +} from '../interfaces/configuration_interfaces.js'; + +/** + * Configuration manager that handles conversion from string-based options + * to proper typed configuration objects. + * + * This is the ONLY place where string parsing should happen for LLM configurations. + */ +export class ConfigurationManager { + private static instance: ConfigurationManager | null = null; + private cachedConfig: AIConfig | null = null; + private lastConfigUpdate: number = 0; + + // Cache for 5 minutes to avoid excessive option reads + private static readonly CACHE_DURATION = 5 * 60 * 1000; + + private constructor() {} + + public static getInstance(): ConfigurationManager { + if (!ConfigurationManager.instance) { + ConfigurationManager.instance = new ConfigurationManager(); + } + return ConfigurationManager.instance; + } + + /** + * Get the complete AI configuration + */ + public async getAIConfig(): Promise { + const now = Date.now(); + if (this.cachedConfig && (now - this.lastConfigUpdate) < ConfigurationManager.CACHE_DURATION) { + return this.cachedConfig; + } + + try { + const config: AIConfig = { + enabled: await this.getAIEnabled(), + providerPrecedence: await this.getProviderPrecedence(), + embeddingProviderPrecedence: await this.getEmbeddingProviderPrecedence(), + defaultModels: await this.getDefaultModels(), + providerSettings: await this.getProviderSettings() + }; + + this.cachedConfig = config; + this.lastConfigUpdate = now; + return config; + } catch (error) { + log.error(`Error loading AI configuration: ${error}`); + return this.getDefaultConfig(); + } + } + + /** + * Parse provider precedence from string option + */ + public async getProviderPrecedence(): Promise { + try { + const precedenceOption = await options.getOption('aiProviderPrecedence'); + const providers = this.parseProviderList(precedenceOption); + + return { + providers: providers as ProviderType[], + defaultProvider: providers[0] as ProviderType + }; + } catch (error) { + log.error(`Error parsing provider precedence: ${error}`); + return { + providers: ['openai', 'anthropic', 'ollama'], + defaultProvider: 'openai' + }; + } + } + + /** + * Parse embedding provider precedence from string option + */ + public async getEmbeddingProviderPrecedence(): Promise { + try { + const precedenceOption = await options.getOption('embeddingProviderPrecedence'); + const providers = this.parseProviderList(precedenceOption); + + return { + providers: providers as EmbeddingProviderType[], + defaultProvider: providers[0] as EmbeddingProviderType + }; + } catch (error) { + log.error(`Error parsing embedding provider precedence: ${error}`); + return { + providers: ['openai', 'ollama'], + defaultProvider: 'openai' + }; + } + } + + /** + * Parse model identifier with optional provider prefix + * Handles formats like "gpt-4", "openai:gpt-4", "ollama:llama2:7b" + */ + public parseModelIdentifier(modelString: string): ModelIdentifier { + if (!modelString) { + return { + modelId: '', + fullIdentifier: '' + }; + } + + const parts = modelString.split(':'); + + if (parts.length === 1) { + // No provider prefix, just model name + return { + modelId: modelString, + fullIdentifier: modelString + }; + } + + // Check if first part is a known provider + const potentialProvider = parts[0].toLowerCase(); + const knownProviders: ProviderType[] = ['openai', 'anthropic', 'ollama']; + + if (knownProviders.includes(potentialProvider as ProviderType)) { + // Provider prefix format + const provider = potentialProvider as ProviderType; + const modelId = parts.slice(1).join(':'); // Rejoin in case model has colons + + return { + provider, + modelId, + fullIdentifier: modelString + }; + } + + // Not a provider prefix, treat whole string as model name + return { + modelId: modelString, + fullIdentifier: modelString + }; + } + + /** + * Create model configuration from string + */ + public createModelConfig(modelString: string, defaultProvider?: ProviderType): ModelConfig { + const identifier = this.parseModelIdentifier(modelString); + const provider = identifier.provider || defaultProvider || 'openai'; + + return { + provider, + modelId: identifier.modelId, + displayName: identifier.fullIdentifier + }; + } + + /** + * Get default models for each provider + */ + public async getDefaultModels(): Promise> { + try { + const [openaiModel, anthropicModel, ollamaModel] = await Promise.all([ + options.getOption('openaiDefaultModel'), + options.getOption('anthropicDefaultModel'), + options.getOption('ollamaDefaultModel') + ]); + + return { + openai: openaiModel || 'gpt-3.5-turbo', + anthropic: anthropicModel || 'claude-3-sonnet-20240229', + ollama: ollamaModel || 'llama2' + }; + } catch (error) { + log.error(`Error loading default models: ${error}`); + return { + openai: 'gpt-3.5-turbo', + anthropic: 'claude-3-sonnet-20240229', + ollama: 'llama2' + }; + } + } + + /** + * Get provider-specific settings + */ + public async getProviderSettings(): Promise { + try { + const [ + openaiApiKey, openaiBaseUrl, openaiDefaultModel, + anthropicApiKey, anthropicBaseUrl, anthropicDefaultModel, + ollamaBaseUrl, ollamaDefaultModel + ] = await Promise.all([ + options.getOption('openaiApiKey'), + options.getOption('openaiBaseUrl'), + options.getOption('openaiDefaultModel'), + options.getOption('anthropicApiKey'), + options.getOption('anthropicBaseUrl'), + options.getOption('anthropicDefaultModel'), + options.getOption('ollamaBaseUrl'), + options.getOption('ollamaDefaultModel') + ]); + + const settings: ProviderSettings = {}; + + if (openaiApiKey || openaiBaseUrl || openaiDefaultModel) { + settings.openai = { + apiKey: openaiApiKey, + baseUrl: openaiBaseUrl, + defaultModel: openaiDefaultModel + }; + } + + if (anthropicApiKey || anthropicBaseUrl || anthropicDefaultModel) { + settings.anthropic = { + apiKey: anthropicApiKey, + baseUrl: anthropicBaseUrl, + defaultModel: anthropicDefaultModel + }; + } + + if (ollamaBaseUrl || ollamaDefaultModel) { + settings.ollama = { + baseUrl: ollamaBaseUrl, + defaultModel: ollamaDefaultModel + }; + } + + return settings; + } catch (error) { + log.error(`Error loading provider settings: ${error}`); + return {}; + } + } + + /** + * Validate configuration + */ + public async validateConfig(): Promise { + const result: ConfigValidationResult = { + isValid: true, + errors: [], + warnings: [] + }; + + try { + const config = await this.getAIConfig(); + + if (!config.enabled) { + result.warnings.push('AI features are disabled'); + return result; + } + + // Validate provider precedence + if (config.providerPrecedence.providers.length === 0) { + result.errors.push('No providers configured in precedence list'); + result.isValid = false; + } + + // Validate provider settings + for (const provider of config.providerPrecedence.providers) { + const providerConfig = config.providerSettings[provider]; + + if (provider === 'openai') { + const openaiConfig = providerConfig as OpenAISettings | undefined; + if (!openaiConfig?.apiKey) { + result.warnings.push('OpenAI API key is not configured'); + } + } + + if (provider === 'anthropic') { + const anthropicConfig = providerConfig as AnthropicSettings | undefined; + if (!anthropicConfig?.apiKey) { + result.warnings.push('Anthropic API key is not configured'); + } + } + + if (provider === 'ollama') { + const ollamaConfig = providerConfig as OllamaSettings | undefined; + if (!ollamaConfig?.baseUrl) { + result.warnings.push('Ollama base URL is not configured'); + } + } + } + + } catch (error) { + result.errors.push(`Configuration validation error: ${error}`); + result.isValid = false; + } + + return result; + } + + /** + * Clear cached configuration (force reload on next access) + */ + public clearCache(): void { + this.cachedConfig = null; + this.lastConfigUpdate = 0; + } + + // Private helper methods + + private async getAIEnabled(): Promise { + try { + return await options.getOptionBool('aiEnabled'); + } catch { + return false; + } + } + + private parseProviderList(precedenceOption: string | null): string[] { + if (!precedenceOption) { + return ['openai', 'anthropic', 'ollama']; + } + + try { + // Handle JSON array format + if (precedenceOption.startsWith('[') && precedenceOption.endsWith(']')) { + const parsed = JSON.parse(precedenceOption); + if (Array.isArray(parsed)) { + return parsed.map(p => String(p).trim()); + } + } + + // Handle comma-separated format + if (precedenceOption.includes(',')) { + return precedenceOption.split(',').map(p => p.trim()); + } + + // Handle single provider + return [precedenceOption.trim()]; + + } catch (error) { + log.error(`Error parsing provider list "${precedenceOption}": ${error}`); + return ['openai', 'anthropic', 'ollama']; + } + } + + private getDefaultConfig(): AIConfig { + return { + enabled: false, + providerPrecedence: { + providers: ['openai', 'anthropic', 'ollama'], + defaultProvider: 'openai' + }, + embeddingProviderPrecedence: { + providers: ['openai', 'ollama'], + defaultProvider: 'openai' + }, + defaultModels: { + openai: 'gpt-3.5-turbo', + anthropic: 'claude-3-sonnet-20240229', + ollama: 'llama2' + }, + providerSettings: {} + }; + } +} + +// Export singleton instance +export default ConfigurationManager.getInstance(); diff --git a/apps/server/src/services/llm/interfaces/configuration_interfaces.ts b/apps/server/src/services/llm/interfaces/configuration_interfaces.ts new file mode 100644 index 000000000..080f373fe --- /dev/null +++ b/apps/server/src/services/llm/interfaces/configuration_interfaces.ts @@ -0,0 +1,108 @@ +/** + * Configuration interfaces for LLM services + * These interfaces replace string parsing with proper typed objects + */ + +/** + * Provider precedence configuration + */ +export interface ProviderPrecedenceConfig { + providers: ProviderType[]; + defaultProvider?: ProviderType; +} + +/** + * Model configuration with provider information + */ +export interface ModelConfig { + provider: ProviderType; + modelId: string; + displayName?: string; + capabilities?: ModelCapabilities; +} + +/** + * Embedding provider precedence configuration + */ +export interface EmbeddingProviderPrecedenceConfig { + providers: EmbeddingProviderType[]; + defaultProvider?: EmbeddingProviderType; +} + +/** + * Model capabilities + */ +export interface ModelCapabilities { + contextWindow?: number; + supportsTools?: boolean; + supportsVision?: boolean; + supportsStreaming?: boolean; + maxTokens?: number; + temperature?: number; +} + +/** + * Complete AI configuration + */ +export interface AIConfig { + enabled: boolean; + providerPrecedence: ProviderPrecedenceConfig; + embeddingProviderPrecedence: EmbeddingProviderPrecedenceConfig; + defaultModels: Record; + providerSettings: ProviderSettings; +} + +/** + * Provider-specific settings + */ +export interface ProviderSettings { + openai?: OpenAISettings; + anthropic?: AnthropicSettings; + ollama?: OllamaSettings; +} + +export interface OpenAISettings { + apiKey?: string; + baseUrl?: string; + defaultModel?: string; +} + +export interface AnthropicSettings { + apiKey?: string; + baseUrl?: string; + defaultModel?: string; +} + +export interface OllamaSettings { + baseUrl?: string; + defaultModel?: string; + timeout?: number; +} + +/** + * Valid provider types + */ +export type ProviderType = 'openai' | 'anthropic' | 'ollama'; + +/** + * Valid embedding provider types + */ +export type EmbeddingProviderType = 'openai' | 'ollama' | 'local'; + +/** + * Model identifier with provider prefix (e.g., "openai:gpt-4" or "ollama:llama2") + */ +export interface ModelIdentifier { + provider?: ProviderType; + modelId: string; + fullIdentifier: string; // The complete string representation +} + +/** + * Validation result for configuration + */ +export interface ConfigValidationResult { + isValid: boolean; + errors: string[]; + warnings: string[]; +} \ No newline at end of file diff --git a/apps/server/src/services/llm/pipeline/stages/model_selection_stage.ts b/apps/server/src/services/llm/pipeline/stages/model_selection_stage.ts index 0830b0bb8..427c63653 100644 --- a/apps/server/src/services/llm/pipeline/stages/model_selection_stage.ts +++ b/apps/server/src/services/llm/pipeline/stages/model_selection_stage.ts @@ -3,9 +3,22 @@ import type { ModelSelectionInput } from '../interfaces.js'; import type { ChatCompletionOptions } from '../../ai_interface.js'; import type { ModelMetadata } from '../../providers/provider_options.js'; import log from '../../../log.js'; -import options from '../../../options.js'; import aiServiceManager from '../../ai_service_manager.js'; import { SEARCH_CONSTANTS, MODEL_CAPABILITIES } from "../../constants/search_constants.js"; + +// Import types +import type { ServiceProviders } from '../../interfaces/ai_service_interfaces.js'; + +// Import new configuration system +import { + getProviderPrecedence, + getPreferredProvider, + parseModelIdentifier, + getDefaultModelForProvider, + createModelConfig +} from '../../config/configuration_helpers.js'; +import type { ProviderType } from '../../interfaces/configuration_interfaces.js'; + /** * Pipeline stage for selecting the appropriate LLM model */ @@ -36,15 +49,15 @@ export class ModelSelectionStage extends BasePipelineStage p.trim()); - } else if (providerPrecedence.startsWith('[') && providerPrecedence.endsWith(']')) { - providers = JSON.parse(providerPrecedence); - } else { - providers = [providerPrecedence]; - } + // Use the new configuration helpers - no string parsing! + defaultProvider = await getPreferredProvider(); + defaultModelName = await getDefaultModelForProvider(defaultProvider); - // Check for first available provider - if (providers.length > 0) { - const firstProvider = providers[0]; - defaultProvider = firstProvider; - - // Get provider-specific default model - if (firstProvider === 'openai') { - const model = await options.getOption('openaiDefaultModel'); - if (model) defaultModelName = model; - } else if (firstProvider === 'anthropic') { - const model = await options.getOption('anthropicDefaultModel'); - if (model) defaultModelName = model; - } else if (firstProvider === 'ollama') { - const model = await options.getOption('ollamaDefaultModel'); - if (model) { - defaultModelName = model; - - // Enable tools for all Ollama models - // The Ollama API will handle models that don't support tool calling - log.info(`Using Ollama model ${model} with tool calling enabled`); - updatedOptions.enableTools = true; - } - } - } - } + log.info(`Selected provider: ${defaultProvider}, model: ${defaultModelName}`); } catch (error) { // If any error occurs, use the fallback default log.error(`Error determining default model: ${error}`); + defaultProvider = 'openai'; + defaultModelName = 'gpt-3.5-turbo'; } // Determine query complexity @@ -162,7 +144,7 @@ export class ModelSelectionStage extends BasePipelineStage { + try { + // Use the new configuration system + const providers = await getProviderPrecedence(); - // Use only providers that are available - const availableProviders = providerPrecedence.filter(provider => - aiServiceManager.isProviderAvailable(provider)); + // Use only providers that are available + const availableProviders = providers.filter(provider => + aiServiceManager.isProviderAvailable(provider)); - if (availableProviders.length === 0) { - throw new Error('No AI providers are available'); + if (availableProviders.length === 0) { + throw new Error('No AI providers are available'); + } + + // Get the first available provider and its default model + const defaultProvider = availableProviders[0]; + const defaultModel = await getDefaultModelForProvider(defaultProvider); + + // Set provider metadata + if (!input.options.providerMetadata) { + input.options.providerMetadata = { + provider: defaultProvider as 'openai' | 'anthropic' | 'ollama' | 'local', + modelId: defaultModel + }; + } + + log.info(`Selected default model ${defaultModel} from provider ${defaultProvider}`); + return defaultModel; + } catch (error) { + log.error(`Error determining default model: ${error}`); + // Fallback to hardcoded default + return 'gpt-3.5-turbo'; } - - // Get the first available provider and its default model - const defaultProvider = availableProviders[0] as 'openai' | 'anthropic' | 'ollama' | 'local'; - let defaultModel = 'gpt-3.5-turbo'; // Use model from our constants - - // Set provider metadata - if (!input.options.providerMetadata) { - input.options.providerMetadata = { - provider: defaultProvider, - modelId: defaultModel - }; - } - - log.info(`Selected default model ${defaultModel} from provider ${defaultProvider}`); - return defaultModel; } /** From ce7c4a31a1e13b585f382134a8edf466128213b5 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Mon, 2 Jun 2025 21:43:36 +0000 Subject: [PATCH 05/18] refactor(llm): enhance configuration handling to avoid default assumptions and improve error handling --- .../src/services/llm/ai_service_manager.ts | 8 +- .../llm/config/configuration_helpers.ts | 59 +++++++++- .../llm/config/configuration_manager.ts | 51 ++++---- .../interfaces/configuration_interfaces.ts | 4 +- .../pipeline/stages/model_selection_stage.ts | 111 ++++++++++-------- 5 files changed, 150 insertions(+), 83 deletions(-) diff --git a/apps/server/src/services/llm/ai_service_manager.ts b/apps/server/src/services/llm/ai_service_manager.ts index b8ec8325e..cee457ebf 100644 --- a/apps/server/src/services/llm/ai_service_manager.ts +++ b/apps/server/src/services/llm/ai_service_manager.ts @@ -532,7 +532,13 @@ export class AIServiceManager implements IAIServiceManager { */ async getPreferredProviderAsync(): Promise { try { - return await getPreferredProvider(); + const preferredProvider = await getPreferredProvider(); + if (preferredProvider === null) { + // No providers configured, fallback to first available + log.info('No providers configured in precedence, using first available provider'); + return this.providerOrder[0]; + } + return preferredProvider; } catch (error) { log.error(`Error getting preferred provider: ${error}`); return this.providerOrder[0]; diff --git a/apps/server/src/services/llm/config/configuration_helpers.ts b/apps/server/src/services/llm/config/configuration_helpers.ts index b4a9d5432..88d2cf1da 100644 --- a/apps/server/src/services/llm/config/configuration_helpers.ts +++ b/apps/server/src/services/llm/config/configuration_helpers.ts @@ -23,8 +23,11 @@ export async function getProviderPrecedence(): Promise { /** * Get the default/preferred AI provider */ -export async function getPreferredProvider(): Promise { +export async function getPreferredProvider(): Promise { const config = await configurationManager.getProviderPrecedence(); + if (config.providers.length === 0) { + return null; // No providers configured + } return config.defaultProvider || config.providers[0]; } @@ -39,8 +42,11 @@ export async function getEmbeddingProviderPrecedence(): Promise { /** * Get the default embedding provider */ -export async function getPreferredEmbeddingProvider(): Promise { +export async function getPreferredEmbeddingProvider(): Promise { const config = await configurationManager.getEmbeddingProviderPrecedence(); + if (config.providers.length === 0) { + return null; // No providers configured + } return config.defaultProvider || config.providers[0]; } @@ -61,9 +67,9 @@ export function createModelConfig(modelString: string, defaultProvider?: Provide /** * Get the default model for a specific provider */ -export async function getDefaultModelForProvider(provider: ProviderType): Promise { +export async function getDefaultModelForProvider(provider: ProviderType): Promise { const config = await configurationManager.getAIConfig(); - return config.defaultModels[provider]; + return config.defaultModels[provider]; // This can now be undefined } /** @@ -106,13 +112,17 @@ export async function isProviderConfigured(provider: ProviderType): Promise { const providers = await getProviderPrecedence(); + if (providers.length === 0) { + return null; // No providers configured + } + for (const provider of providers) { if (await isProviderConfigured(provider)) { return provider; } } - return null; + return null; // No providers are properly configured } /** @@ -128,3 +138,42 @@ export async function validateConfiguration() { export function clearConfigurationCache(): void { configurationManager.clearCache(); } + +/** + * Get a model configuration with validation that no defaults are assumed + */ +export async function getValidModelConfig(provider: ProviderType): Promise<{ model: string; provider: ProviderType } | null> { + const defaultModel = await getDefaultModelForProvider(provider); + + if (!defaultModel) { + // No default model configured for this provider + return null; + } + + const isConfigured = await isProviderConfigured(provider); + if (!isConfigured) { + // Provider is not properly configured + return null; + } + + return { + model: defaultModel, + provider + }; +} + +/** + * Get the first valid model configuration from the provider precedence list + */ +export async function getFirstValidModelConfig(): Promise<{ model: string; provider: ProviderType } | null> { + const providers = await getProviderPrecedence(); + + for (const provider of providers) { + const config = await getValidModelConfig(provider); + if (config) { + return config; + } + } + + return null; // No valid model configuration found +} diff --git a/apps/server/src/services/llm/config/configuration_manager.ts b/apps/server/src/services/llm/config/configuration_manager.ts index 621e5d3f4..5bc9611b8 100644 --- a/apps/server/src/services/llm/config/configuration_manager.ts +++ b/apps/server/src/services/llm/config/configuration_manager.ts @@ -75,13 +75,14 @@ export class ConfigurationManager { return { providers: providers as ProviderType[], - defaultProvider: providers[0] as ProviderType + defaultProvider: providers.length > 0 ? providers[0] as ProviderType : undefined }; } catch (error) { log.error(`Error parsing provider precedence: ${error}`); + // Only return known providers if they exist, don't assume defaults return { - providers: ['openai', 'anthropic', 'ollama'], - defaultProvider: 'openai' + providers: [], + defaultProvider: undefined }; } } @@ -96,13 +97,14 @@ export class ConfigurationManager { return { providers: providers as EmbeddingProviderType[], - defaultProvider: providers[0] as EmbeddingProviderType + defaultProvider: providers.length > 0 ? providers[0] as EmbeddingProviderType : undefined }; } catch (error) { log.error(`Error parsing embedding provider precedence: ${error}`); + // Don't assume defaults, return empty configuration return { - providers: ['openai', 'ollama'], - defaultProvider: 'openai' + providers: [], + defaultProvider: undefined }; } } @@ -167,9 +169,9 @@ export class ConfigurationManager { } /** - * Get default models for each provider + * Get default models for each provider - ONLY from user configuration */ - public async getDefaultModels(): Promise> { + public async getDefaultModels(): Promise> { try { const [openaiModel, anthropicModel, ollamaModel] = await Promise.all([ options.getOption('openaiDefaultModel'), @@ -178,16 +180,17 @@ export class ConfigurationManager { ]); return { - openai: openaiModel || 'gpt-3.5-turbo', - anthropic: anthropicModel || 'claude-3-sonnet-20240229', - ollama: ollamaModel || 'llama2' + openai: openaiModel || undefined, + anthropic: anthropicModel || undefined, + ollama: ollamaModel || undefined }; } catch (error) { log.error(`Error loading default models: ${error}`); + // Return undefined for all providers if we can't load config return { - openai: 'gpt-3.5-turbo', - anthropic: 'claude-3-sonnet-20240229', - ollama: 'llama2' + openai: undefined, + anthropic: undefined, + ollama: undefined }; } } @@ -322,7 +325,8 @@ export class ConfigurationManager { private parseProviderList(precedenceOption: string | null): string[] { if (!precedenceOption) { - return ['openai', 'anthropic', 'ollama']; + // Don't assume any defaults - return empty array + return []; } try { @@ -344,7 +348,8 @@ export class ConfigurationManager { } catch (error) { log.error(`Error parsing provider list "${precedenceOption}": ${error}`); - return ['openai', 'anthropic', 'ollama']; + // Don't assume defaults on parse error + return []; } } @@ -352,17 +357,17 @@ export class ConfigurationManager { return { enabled: false, providerPrecedence: { - providers: ['openai', 'anthropic', 'ollama'], - defaultProvider: 'openai' + providers: [], + defaultProvider: undefined }, embeddingProviderPrecedence: { - providers: ['openai', 'ollama'], - defaultProvider: 'openai' + providers: [], + defaultProvider: undefined }, defaultModels: { - openai: 'gpt-3.5-turbo', - anthropic: 'claude-3-sonnet-20240229', - ollama: 'llama2' + openai: undefined, + anthropic: undefined, + ollama: undefined }, providerSettings: {} }; diff --git a/apps/server/src/services/llm/interfaces/configuration_interfaces.ts b/apps/server/src/services/llm/interfaces/configuration_interfaces.ts index 080f373fe..5a03dc4f1 100644 --- a/apps/server/src/services/llm/interfaces/configuration_interfaces.ts +++ b/apps/server/src/services/llm/interfaces/configuration_interfaces.ts @@ -48,7 +48,7 @@ export interface AIConfig { enabled: boolean; providerPrecedence: ProviderPrecedenceConfig; embeddingProviderPrecedence: EmbeddingProviderPrecedenceConfig; - defaultModels: Record; + defaultModels: Record; providerSettings: ProviderSettings; } @@ -105,4 +105,4 @@ export interface ConfigValidationResult { isValid: boolean; errors: string[]; warnings: string[]; -} \ No newline at end of file +} diff --git a/apps/server/src/services/llm/pipeline/stages/model_selection_stage.ts b/apps/server/src/services/llm/pipeline/stages/model_selection_stage.ts index 427c63653..fdecc216e 100644 --- a/apps/server/src/services/llm/pipeline/stages/model_selection_stage.ts +++ b/apps/server/src/services/llm/pipeline/stages/model_selection_stage.ts @@ -100,61 +100,65 @@ export class ModelSelectionStage extends BasePipelineStage query.toLowerCase().includes(term)); - const isLongQuery = query.length > 100; - const hasMultipleQuestions = (query.match(/\?/g) || []).length > 1; - - if ((hasComplexTerms && isLongQuery) || hasMultipleQuestions) { - queryComplexity = 'high'; - } else if (hasComplexTerms || isLongQuery) { - queryComplexity = 'medium'; + if (!preferredProvider) { + throw new Error('No AI providers are configured. Please check your AI settings.'); } + + const modelName = await getDefaultModelForProvider(preferredProvider); + + if (!modelName) { + throw new Error(`No default model configured for provider ${preferredProvider}. Please set a default model in your AI settings.`); + } + + log.info(`Selected provider: ${preferredProvider}, model: ${modelName}`); + + // Determine query complexity + let queryComplexity = 'low'; + if (query) { + // Simple heuristic: longer queries or those with complex terms indicate higher complexity + const complexityIndicators = [ + 'explain', 'analyze', 'compare', 'evaluate', 'synthesize', + 'summarize', 'elaborate', 'investigate', 'research', 'debate' + ]; + + const hasComplexTerms = complexityIndicators.some(term => query.toLowerCase().includes(term)); + const isLongQuery = query.length > 100; + const hasMultipleQuestions = (query.match(/\?/g) || []).length > 1; + + if ((hasComplexTerms && isLongQuery) || hasMultipleQuestions) { + queryComplexity = 'high'; + } else if (hasComplexTerms || isLongQuery) { + queryComplexity = 'medium'; + } + } + + // Check content length if provided + if (contentLength && contentLength > SEARCH_CONSTANTS.CONTEXT.CONTENT_LENGTH.MEDIUM_THRESHOLD) { + // For large content, favor more powerful models + queryComplexity = contentLength > SEARCH_CONSTANTS.CONTEXT.CONTENT_LENGTH.HIGH_THRESHOLD ? 'high' : 'medium'; + } + + // Set the model and add provider metadata + updatedOptions.model = modelName; + this.addProviderMetadata(updatedOptions, preferredProvider as ServiceProviders, modelName); + + log.info(`Selected model: ${modelName} from provider: ${preferredProvider} for query complexity: ${queryComplexity}`); + log.info(`[ModelSelectionStage] Final options: ${JSON.stringify({ + model: updatedOptions.model, + stream: updatedOptions.stream, + provider: preferredProvider, + enableTools: updatedOptions.enableTools + })}`); + + return { options: updatedOptions }; + } catch (error) { + log.error(`Error determining default model: ${error}`); + throw new Error(`Failed to determine AI model configuration: ${error}`); } - - // Check content length if provided - if (contentLength && contentLength > SEARCH_CONSTANTS.CONTEXT.CONTENT_LENGTH.MEDIUM_THRESHOLD) { - // For large content, favor more powerful models - queryComplexity = contentLength > SEARCH_CONSTANTS.CONTEXT.CONTENT_LENGTH.HIGH_THRESHOLD ? 'high' : 'medium'; - } - - // Set the model and add provider metadata - updatedOptions.model = defaultModelName; - this.addProviderMetadata(updatedOptions, defaultProvider as ServiceProviders, defaultModelName); - - log.info(`Selected model: ${defaultModelName} from provider: ${defaultProvider} for query complexity: ${queryComplexity}`); - log.info(`[ModelSelectionStage] Final options: ${JSON.stringify({ - model: updatedOptions.model, - stream: updatedOptions.stream, - provider: defaultProvider, - enableTools: updatedOptions.enableTools - })}`); - - return { options: updatedOptions }; } /** @@ -225,6 +229,10 @@ export class ModelSelectionStage extends BasePipelineStage Date: Mon, 2 Jun 2025 21:44:43 +0000 Subject: [PATCH 06/18] refactor(llm): streamline model selection by utilizing the new configuration system and enhance error handling --- .../services/llm/chat/rest_chat_service.ts | 51 ++++++------------- 1 file changed, 15 insertions(+), 36 deletions(-) diff --git a/apps/server/src/services/llm/chat/rest_chat_service.ts b/apps/server/src/services/llm/chat/rest_chat_service.ts index 4c1c9d2da..54064060c 100644 --- a/apps/server/src/services/llm/chat/rest_chat_service.ts +++ b/apps/server/src/services/llm/chat/rest_chat_service.ts @@ -20,6 +20,12 @@ import type { NoteSource } from "../interfaces/chat_session.js"; import type { LLMStreamMessage } from "../interfaces/chat_ws_messages.js"; import type { ChatMessage } from '../interfaces/chat_session.js'; import type { ChatSession } from '../interfaces/chat_session.js'; +import { + isAIEnabled, + getFirstValidModelConfig, + getDefaultModelForProvider, + getPreferredProvider +} from '../config/configuration_helpers.js'; /** * Service to handle chat API interactions @@ -705,7 +711,7 @@ class RestChatService { } /** - * Get the user's preferred model + * Get the user's preferred model using the new configuration system */ async getPreferredModel(sessionModel?: string): Promise { // If the session already has a valid model (not 'default'), use it @@ -714,46 +720,19 @@ class RestChatService { } try { - // Get provider precedence list (same logic as model selection stage) - const providerPrecedence = await options.getOption('aiProviderPrecedence'); - let defaultProvider = 'openai'; - let defaultModelName = 'gpt-3.5-turbo'; + // Use the new configuration system - no string parsing! + const validConfig = await getFirstValidModelConfig(); - if (providerPrecedence) { - // Parse provider precedence list - let providers: string[] = []; - if (providerPrecedence.includes(',')) { - providers = providerPrecedence.split(',').map(p => p.trim()); - } else if (providerPrecedence.startsWith('[') && providerPrecedence.endsWith(']')) { - providers = JSON.parse(providerPrecedence); - } else { - providers = [providerPrecedence]; - } - - // Get first available provider - if (providers.length > 0) { - const firstProvider = providers[0]; - defaultProvider = firstProvider; - - // Get provider-specific default model - if (firstProvider === 'openai') { - const model = await options.getOption('openaiDefaultModel'); - if (model) defaultModelName = model; - } else if (firstProvider === 'anthropic') { - const model = await options.getOption('anthropicDefaultModel'); - if (model) defaultModelName = model; - } else if (firstProvider === 'ollama') { - const model = await options.getOption('ollamaDefaultModel'); - if (model) defaultModelName = model; - } - } + if (!validConfig) { + log.error('No valid AI model configuration found. Please configure your AI settings.'); + return undefined; // Don't provide fallback defaults } - log.info(`Selected user's preferred model: ${defaultModelName} from provider: ${defaultProvider}`); - return defaultModelName; + log.info(`Selected user's preferred model: ${validConfig.model} from provider: ${validConfig.provider}`); + return validConfig.model; } catch (error) { log.error(`Error getting user's preferred model: ${error}`); - return 'gpt-3.5-turbo'; // Fallback + return undefined; // Don't provide fallback defaults, let the caller handle it } } } From 3a55735cd5c8d47f0933c63c33e7c41231391255 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Mon, 2 Jun 2025 21:49:35 +0000 Subject: [PATCH 07/18] refactor(llm): implement new configuration methods for provider order and validation, enhancing error handling and deprecating legacy functions --- .../src/services/llm/ai_service_manager.ts | 161 +++++++++--------- .../src/services/llm/embeddings/storage.ts | 57 +++---- 2 files changed, 100 insertions(+), 118 deletions(-) diff --git a/apps/server/src/services/llm/ai_service_manager.ts b/apps/server/src/services/llm/ai_service_manager.ts index cee457ebf..a398db0ad 100644 --- a/apps/server/src/services/llm/ai_service_manager.ts +++ b/apps/server/src/services/llm/ai_service_manager.ts @@ -26,7 +26,8 @@ import { parseModelIdentifier, isAIEnabled, getDefaultModelForProvider, - clearConfigurationCache + clearConfigurationCache, + validateConfiguration } from './config/configuration_helpers.js'; import type { ProviderType } from './interfaces/configuration_interfaces.js'; @@ -48,7 +49,7 @@ export class AIServiceManager implements IAIServiceManager { ollama: new OllamaService() }; - private providerOrder: ServiceProviders[] = ['openai', 'anthropic', 'ollama']; // Default order + private providerOrder: ServiceProviders[] = []; // Will be populated from configuration private initialized = false; constructor() { @@ -84,6 +85,23 @@ export class AIServiceManager implements IAIServiceManager { /** * Update the provider precedence order using the new configuration system + */ + async updateProviderOrderAsync(): Promise { + try { + const providers = await getProviderPrecedence(); + this.providerOrder = providers as ServiceProviders[]; + this.initialized = true; + log.info(`Updated provider order: ${providers.join(', ')}`); + } catch (error) { + log.error(`Failed to get provider precedence: ${error}`); + // Keep empty order, will be handled gracefully by other methods + this.providerOrder = []; + this.initialized = true; + } + } + + /** + * Update the provider precedence order (legacy sync version) * Returns true if successful, false if options not available yet */ updateProviderOrder(): boolean { @@ -91,89 +109,57 @@ export class AIServiceManager implements IAIServiceManager { return true; } - try { - // Use async helper but handle it synchronously for now - // In a real refactor, this method should become async - getProviderPrecedence().then(providers => { - this.providerOrder = providers as ServiceProviders[]; - log.info(`Updated provider order: ${providers.join(', ')}`); - }).catch(error => { - log.error(`Failed to get provider precedence: ${error}`); - // Keep default order - }); + // Use async version but don't wait + this.updateProviderOrderAsync().catch(error => { + log.error(`Error in async provider order update: ${error}`); + }); - this.initialized = true; - return true; + return true; + } + + /** + * Validate AI configuration using the new configuration system + */ + async validateConfiguration(): Promise { + try { + const result = await validateConfiguration(); + + if (!result.isValid) { + let message = 'There are issues with your AI configuration:'; + for (const error of result.errors) { + message += `\n• ${error}`; + } + if (result.warnings.length > 0) { + message += '\n\nWarnings:'; + for (const warning of result.warnings) { + message += `\n• ${warning}`; + } + } + message += '\n\nPlease check your AI settings.'; + return message; + } + + if (result.warnings.length > 0) { + let message = 'AI configuration warnings:'; + for (const warning of result.warnings) { + message += `\n• ${warning}`; + } + log.info(message); + } + + return null; } catch (error) { - // If options table doesn't exist yet, use defaults - // This happens during initial database creation - this.providerOrder = ['openai', 'anthropic', 'ollama']; - return false; + log.error(`Error validating AI configuration: ${error}`); + return `Configuration validation failed: ${error}`; } } /** - * Validate embedding providers configuration using the new configuration system + * @deprecated Use validateConfiguration() instead */ async validateEmbeddingProviders(): Promise { - try { - // Check if AI is enabled using the new helper - const aiEnabled = await isAIEnabled(); - if (!aiEnabled) { - return null; - } - - // Get precedence list using the new helper (no string parsing!) - const precedenceList = await getEmbeddingProviderPrecedence(); - - // Check for configuration issues with providers in the precedence list - const configIssues: string[] = []; - - // Check each provider in the precedence list for proper configuration - for (const provider of precedenceList) { - if (provider === 'openai') { - // Check OpenAI configuration - const apiKey = await options.getOption('openaiApiKey'); - if (!apiKey) { - configIssues.push(`OpenAI API key is missing`); - } - } else if (provider === 'anthropic') { - // Check Anthropic configuration - const apiKey = await options.getOption('anthropicApiKey'); - if (!apiKey) { - configIssues.push(`Anthropic API key is missing`); - } - } else if (provider === 'ollama') { - // Check Ollama configuration - const baseUrl = await options.getOption('ollamaBaseUrl'); - if (!baseUrl) { - configIssues.push(`Ollama Base URL is missing`); - } - } - // Add checks for other providers as needed - } - - // Return warning message if there are configuration issues - if (configIssues.length > 0) { - let message = 'There are issues with your AI provider configuration:'; - - for (const issue of configIssues) { - message += `\n• ${issue}`; - } - - message += '\n\nPlease check your AI settings.'; - - // Log warning to console - log.error('AI Provider Configuration Warning: ' + message); - - return message; - } - - return null; - } catch (error) { - log.error(`Error validating embedding providers: ${error}`); - return null; - } + log.info('validateEmbeddingProviders is deprecated, use validateConfiguration instead'); + return this.validateConfiguration(); } /** @@ -348,6 +334,13 @@ export class AIServiceManager implements IAIServiceManager { /** * Get whether AI features are enabled using the new configuration system */ + async getAIEnabledAsync(): Promise { + return isAIEnabled(); + } + + /** + * Get whether AI features are enabled (sync version for compatibility) + */ getAIEnabled(): boolean { // For synchronous compatibility, use the old method // In a full refactor, this should be async @@ -355,11 +348,12 @@ export class AIServiceManager implements IAIServiceManager { } /** - * Set up embeddings provider for AI features + * Set up embeddings provider using the new configuration system */ async setupEmbeddingsProvider(): Promise { try { - if (!this.getAIEnabled()) { + const aiEnabled = await isAIEnabled(); + if (!aiEnabled) { log.info('AI features are disabled'); return; } @@ -381,20 +375,23 @@ export class AIServiceManager implements IAIServiceManager { } /** - * Initialize the AI Service + * Initialize the AI Service using the new configuration system */ async initialize(): Promise { try { log.info("Initializing AI service..."); // Check if AI is enabled using the new helper - const isAIEnabled_value = await isAIEnabled(); + const aiEnabled = await isAIEnabled(); - if (!isAIEnabled_value) { + if (!aiEnabled) { log.info("AI features are disabled in options"); return; } + // Update provider order from configuration + await this.updateProviderOrderAsync(); + // Set up embeddings provider if AI is enabled await this.setupEmbeddingsProvider(); diff --git a/apps/server/src/services/llm/embeddings/storage.ts b/apps/server/src/services/llm/embeddings/storage.ts index 01cc2ac17..ac096071f 100644 --- a/apps/server/src/services/llm/embeddings/storage.ts +++ b/apps/server/src/services/llm/embeddings/storage.ts @@ -1,4 +1,4 @@ -import sql from "../../sql.js"; +import sql from '../../sql.js' import { randomString } from "../../../services/utils.js"; import dateUtils from "../../../services/date_utils.js"; import log from "../../log.js"; @@ -11,6 +11,7 @@ import { SEARCH_CONSTANTS } from '../constants/search_constants.js'; import type { NoteEmbeddingContext } from "./embeddings_interface.js"; import becca from "../../../becca/becca.js"; import { isNoteExcludedFromAIById } from "../utils/ai_exclusion_utils.js"; +import { getEmbeddingProviderPrecedence } from '../config/configuration_helpers.js'; interface Similarity { noteId: string; @@ -271,44 +272,28 @@ export async function findSimilarNotes( } } } else { - // Use dedicated embedding provider precedence from options for other strategies - let preferredProviders: string[] = []; - const embeddingPrecedence = await options.getOption('embeddingProviderPrecedence'); + // Try providers using the new configuration system + if (useFallback) { + log.info('No embeddings found for specified provider, trying fallback providers...'); - if (embeddingPrecedence) { - // For "comma,separated,values" - if (embeddingPrecedence.includes(',')) { - preferredProviders = embeddingPrecedence.split(',').map(p => p.trim()); - } - // For JSON array ["value1", "value2"] - else if (embeddingPrecedence.startsWith('[') && embeddingPrecedence.endsWith(']')) { - try { - preferredProviders = JSON.parse(embeddingPrecedence); - } catch (e) { - log.error(`Error parsing embedding precedence: ${e}`); - preferredProviders = [embeddingPrecedence]; // Fallback to using as single value + // Use the new configuration system - no string parsing! + const preferredProviders = await getEmbeddingProviderPrecedence(); + + log.info(`Using provider precedence: ${preferredProviders.join(', ')}`); + + // Try providers in precedence order + for (const provider of preferredProviders) { + const providerEmbeddings = availableEmbeddings.filter(e => e.providerId === provider); + + if (providerEmbeddings.length > 0) { + // Choose the model with the most embeddings + const bestModel = providerEmbeddings.sort((a, b) => b.count - a.count)[0]; + log.info(`Found fallback provider: ${provider}, model: ${bestModel.modelId}, dimension: ${bestModel.dimension}`); + + // The 'regenerate' strategy would go here if needed + // We're no longer supporting the 'adapt' strategy } } - // For a single value - else { - preferredProviders = [embeddingPrecedence]; - } - } - - log.info(`Using provider precedence: ${preferredProviders.join(', ')}`); - - // Try providers in precedence order - for (const provider of preferredProviders) { - const providerEmbeddings = availableEmbeddings.filter(e => e.providerId === provider); - - if (providerEmbeddings.length > 0) { - // Choose the model with the most embeddings - const bestModel = providerEmbeddings.sort((a, b) => b.count - a.count)[0]; - log.info(`Found fallback provider: ${provider}, model: ${bestModel.modelId}, dimension: ${bestModel.dimension}`); - - // The 'regenerate' strategy would go here if needed - // We're no longer supporting the 'adapt' strategy - } } } } From d8d5318ace36b5b878efc5a4a02adff5950fa966 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Mon, 2 Jun 2025 21:55:52 +0000 Subject: [PATCH 08/18] refactor(llm): remove deprecated validateEmbeddingProviders method and update session handling in chat services --- apps/server/src/services/llm/ai_service_manager.ts | 10 +--------- apps/server/src/services/llm/chat/rest_chat_service.ts | 5 ----- apps/server/src/services/llm/chat/sessions_store.ts | 1 - 3 files changed, 1 insertion(+), 15 deletions(-) diff --git a/apps/server/src/services/llm/ai_service_manager.ts b/apps/server/src/services/llm/ai_service_manager.ts index a398db0ad..d7bbf4cf7 100644 --- a/apps/server/src/services/llm/ai_service_manager.ts +++ b/apps/server/src/services/llm/ai_service_manager.ts @@ -154,14 +154,6 @@ export class AIServiceManager implements IAIServiceManager { } } - /** - * @deprecated Use validateConfiguration() instead - */ - async validateEmbeddingProviders(): Promise { - log.info('validateEmbeddingProviders is deprecated, use validateConfiguration instead'); - return this.validateConfiguration(); - } - /** * Ensure manager is initialized before using */ @@ -626,7 +618,7 @@ export default { }, // Add validateEmbeddingProviders method async validateEmbeddingProviders(): Promise { - return getInstance().validateEmbeddingProviders(); + return getInstance().validateConfiguration(); }, // Context and index related methods getContextExtractor() { diff --git a/apps/server/src/services/llm/chat/rest_chat_service.ts b/apps/server/src/services/llm/chat/rest_chat_service.ts index 54064060c..5693184b1 100644 --- a/apps/server/src/services/llm/chat/rest_chat_service.ts +++ b/apps/server/src/services/llm/chat/rest_chat_service.ts @@ -139,13 +139,8 @@ class RestChatService { if (!session && req.method === 'POST') { log.info(`No Chat Note found for ${chatNoteId}, creating a new Chat Note and session`); - // Create a new Chat Note via the storage service - //const chatStorageService = (await import('../../llm/chat_storage_service.js')).default; - //const newChat = await chatStorageService.createChat('New Chat'); - // Use the new Chat Note's ID for the session session = SessionsStore.createSession({ - //title: newChat.title, chatNoteId: chatNoteId }); diff --git a/apps/server/src/services/llm/chat/sessions_store.ts b/apps/server/src/services/llm/chat/sessions_store.ts index 65715ab23..c16b1af04 100644 --- a/apps/server/src/services/llm/chat/sessions_store.ts +++ b/apps/server/src/services/llm/chat/sessions_store.ts @@ -4,7 +4,6 @@ import log from "../../log.js"; import { LLM_CONSTANTS } from '../constants/provider_constants.js'; import { SEARCH_CONSTANTS } from '../constants/search_constants.js'; -import { randomString } from "../../utils.js"; import type { ChatSession, ChatMessage } from '../interfaces/chat_session.js'; // In-memory storage for sessions From ed64a5b4f76fcdf9015f44571a314579ab8308d8 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Mon, 2 Jun 2025 22:09:59 +0000 Subject: [PATCH 09/18] refactor(llm): simplify chat handling by removing session store and directly integrating chat storage service --- apps/server/src/routes/api/llm.ts | 52 +- .../llm/chat/handlers/tool_handler.ts | 11 - apps/server/src/services/llm/chat/index.ts | 2 - .../services/llm/chat/rest_chat_service.ts | 571 ++++-------------- .../src/services/llm/chat/sessions_store.ts | 168 ------ 5 files changed, 151 insertions(+), 653 deletions(-) delete mode 100644 apps/server/src/services/llm/chat/sessions_store.ts diff --git a/apps/server/src/routes/api/llm.ts b/apps/server/src/routes/api/llm.ts index be046bdd6..80bc36884 100644 --- a/apps/server/src/routes/api/llm.ts +++ b/apps/server/src/routes/api/llm.ts @@ -5,7 +5,6 @@ import options from "../../services/options.js"; // Import the index service for knowledge base management import indexService from "../../services/llm/index_service.js"; import restChatService from "../../services/llm/rest_chat_service.js"; -import chatService from '../../services/llm/chat_service.js'; import chatStorageService from '../../services/llm/chat_storage_service.js'; // Define basic interfaces @@ -190,23 +189,26 @@ async function getSession(req: Request, res: Response) { * tags: ["llm"] */ async function updateSession(req: Request, res: Response) { - // Get the chat using ChatService + // Get the chat using chatStorageService directly const chatNoteId = req.params.chatNoteId; const updates = req.body; try { // Get the chat - const session = await chatService.getOrCreateSession(chatNoteId); + const chat = await chatStorageService.getChat(chatNoteId); + if (!chat) { + throw new Error(`Chat with ID ${chatNoteId} not found`); + } // Update title if provided if (updates.title) { - await chatStorageService.updateChat(chatNoteId, session.messages, updates.title); + await chatStorageService.updateChat(chatNoteId, chat.messages, updates.title); } // Return the updated chat return { id: chatNoteId, - title: updates.title || session.title, + title: updates.title || chat.title, updatedAt: new Date() }; } catch (error) { @@ -248,18 +250,18 @@ async function updateSession(req: Request, res: Response) { * tags: ["llm"] */ async function listSessions(req: Request, res: Response) { - // Get all sessions using ChatService + // Get all sessions using chatStorageService directly try { - const sessions = await chatService.getAllSessions(); + const chats = await chatStorageService.getAllChats(); // Format the response return { - sessions: sessions.map(session => ({ - id: session.id, - title: session.title, - createdAt: new Date(), // Since we don't have this in chat sessions - lastActive: new Date(), // Since we don't have this in chat sessions - messageCount: session.messages.length + sessions: chats.map(chat => ({ + id: chat.id, + title: chat.title, + createdAt: chat.createdAt || new Date(), + lastActive: chat.updatedAt || new Date(), + messageCount: chat.messages.length })) }; } catch (error) { @@ -814,16 +816,14 @@ async function streamMessage(req: Request, res: Response) { throw new Error('Content cannot be empty'); } - // Get or create session from Chat Note - // This will check the sessions store first, and if not found, create from the Chat Note - const session = await restChatService.getOrCreateSessionFromChatNote(chatNoteId, true); - if (!session) { - throw new Error('Chat not found and could not be created from note'); + // Get or create chat directly from storage (simplified approach) + let chat = await chatStorageService.getChat(chatNoteId); + if (!chat) { + // Create a new chat if it doesn't exist + chat = await chatStorageService.createChat('New Chat'); + log.info(`Created new chat with ID: ${chat.id} for stream request`); } - // Update last active timestamp - session.lastActive = new Date(); - // Process mentions if provided let enhancedContent = content; if (mentions && Array.isArray(mentions) && mentions.length > 0) { @@ -858,13 +858,15 @@ async function streamMessage(req: Request, res: Response) { } } - // Add user message to the session (with enhanced content for processing) - session.messages.push({ + // Add user message to the chat (without timestamp since Message interface doesn't support it) + chat.messages.push({ role: 'user', - content: enhancedContent, - timestamp: new Date() + content: enhancedContent }); + // Save the updated chat + await chatStorageService.updateChat(chat.id, chat.messages, chat.title); + // Create request parameters for the pipeline const requestParams = { chatNoteId: chatNoteId, diff --git a/apps/server/src/services/llm/chat/handlers/tool_handler.ts b/apps/server/src/services/llm/chat/handlers/tool_handler.ts index 076664f63..40520ebe3 100644 --- a/apps/server/src/services/llm/chat/handlers/tool_handler.ts +++ b/apps/server/src/services/llm/chat/handlers/tool_handler.ts @@ -3,7 +3,6 @@ */ import log from "../../../log.js"; import type { Message } from "../../ai_interface.js"; -import SessionsStore from "../sessions_store.js"; /** * Handles the execution of LLM tools @@ -101,11 +100,6 @@ export class ToolHandler { : JSON.stringify(result).substring(0, 100) + '...'; log.info(`Tool result: ${resultPreview}`); - // Record tool execution in session if chatNoteId is provided - if (chatNoteId) { - SessionsStore.recordToolExecution(chatNoteId, toolCall, typeof result === 'string' ? result : JSON.stringify(result)); - } - // Format result as a proper message return { role: 'tool', @@ -116,11 +110,6 @@ export class ToolHandler { } catch (error: any) { log.error(`Error executing tool ${toolCall.function.name}: ${error.message}`); - // Record error in session if chatNoteId is provided - if (chatNoteId) { - SessionsStore.recordToolExecution(chatNoteId, toolCall, '', error.message); - } - // Return error as tool result return { role: 'tool', diff --git a/apps/server/src/services/llm/chat/index.ts b/apps/server/src/services/llm/chat/index.ts index d82554229..79b587a09 100644 --- a/apps/server/src/services/llm/chat/index.ts +++ b/apps/server/src/services/llm/chat/index.ts @@ -2,7 +2,6 @@ * Chat module export */ import restChatService from './rest_chat_service.js'; -import sessionsStore from './sessions_store.js'; import { ContextHandler } from './handlers/context_handler.js'; import { ToolHandler } from './handlers/tool_handler.js'; import { StreamHandler } from './handlers/stream_handler.js'; @@ -13,7 +12,6 @@ import type { LLMStreamMessage } from '../interfaces/chat_ws_messages.js'; // Export components export { restChatService as default, - sessionsStore, ContextHandler, ToolHandler, StreamHandler, diff --git a/apps/server/src/services/llm/chat/rest_chat_service.ts b/apps/server/src/services/llm/chat/rest_chat_service.ts index 5693184b1..b6b0bc820 100644 --- a/apps/server/src/services/llm/chat/rest_chat_service.ts +++ b/apps/server/src/services/llm/chat/rest_chat_service.ts @@ -1,5 +1,6 @@ /** - * Service to handle chat API interactions + * Simplified service to handle chat API interactions + * Works directly with ChatStorageService - no complex session management */ import log from "../../log.js"; import type { Request, Response } from "express"; @@ -8,27 +9,16 @@ import { AIServiceManager } from "../ai_service_manager.js"; import { ChatPipeline } from "../pipeline/chat_pipeline.js"; import type { ChatPipelineInput } from "../pipeline/interfaces.js"; import options from "../../options.js"; -import { SEARCH_CONSTANTS } from '../constants/search_constants.js'; - -// Import our refactored modules -import { ContextHandler } from "./handlers/context_handler.js"; import { ToolHandler } from "./handlers/tool_handler.js"; -import { StreamHandler } from "./handlers/stream_handler.js"; -import SessionsStore from "./sessions_store.js"; -import * as MessageFormatter from "./utils/message_formatter.js"; -import type { NoteSource } from "../interfaces/chat_session.js"; import type { LLMStreamMessage } from "../interfaces/chat_ws_messages.js"; -import type { ChatMessage } from '../interfaces/chat_session.js'; -import type { ChatSession } from '../interfaces/chat_session.js'; +import chatStorageService from '../chat_storage_service.js'; import { isAIEnabled, getFirstValidModelConfig, - getDefaultModelForProvider, - getPreferredProvider } from '../config/configuration_helpers.js'; /** - * Service to handle chat API interactions + * Simplified service to handle chat API interactions */ class RestChatService { /** @@ -47,35 +37,15 @@ class RestChatService { * Check if AI services are available */ safelyUseAIManager(): boolean { - // Only use AI manager if database is initialized if (!this.isDatabaseInitialized()) { log.info("AI check failed: Database is not initialized"); return false; } - // Try to access the manager - will create instance only if needed try { - // Create local instance to avoid circular references const aiManager = new AIServiceManager(); - - if (!aiManager) { - log.info("AI check failed: AI manager module is not available"); - return false; - } - const isAvailable = aiManager.isAnyServiceAvailable(); log.info(`AI service availability check result: ${isAvailable}`); - - if (isAvailable) { - // Additional diagnostics - try { - const providers = aiManager.getAvailableProviders(); - log.info(`Available AI providers: ${providers.join(', ')}`); - } catch (err) { - log.info(`Could not get available providers: ${err}`); - } - } - return isAvailable; } catch (error) { log.error(`Error accessing AI service manager: ${error}`); @@ -85,299 +55,163 @@ class RestChatService { /** * Handle a message sent to an LLM and get a response + * Simplified to work directly with chat storage */ async handleSendMessage(req: Request, res: Response) { - log.info("=== Starting handleSendMessage ==="); + log.info("=== Starting simplified handleSendMessage ==="); try { - // Extract parameters differently based on the request method + // Extract parameters let content, useAdvancedContext, showThinking, chatNoteId; if (req.method === 'POST') { - // For POST requests, get content from the request body const requestBody = req.body || {}; content = requestBody.content; useAdvancedContext = requestBody.useAdvancedContext || false; showThinking = requestBody.showThinking || false; - - // Add logging for POST requests - log.info(`LLM POST message: chatNoteId=${req.params.chatNoteId}, useAdvancedContext=${useAdvancedContext}, showThinking=${showThinking}, contentLength=${content ? content.length : 0}`); + log.info(`LLM POST message: chatNoteId=${req.params.chatNoteId}, contentLength=${content ? content.length : 0}`); } else if (req.method === 'GET') { - // For GET (streaming) requests, get parameters from query params and body - // For streaming requests, we need the content from the body useAdvancedContext = req.query.useAdvancedContext === 'true' || (req.body && req.body.useAdvancedContext === true); showThinking = req.query.showThinking === 'true' || (req.body && req.body.showThinking === true); content = req.body && req.body.content ? req.body.content : ''; - - // Add detailed logging for GET requests - log.info(`LLM GET stream: chatNoteId=${req.params.chatNoteId}, useAdvancedContext=${useAdvancedContext}, showThinking=${showThinking}`); - log.info(`Parameters from query: useAdvancedContext=${req.query.useAdvancedContext}, showThinking=${req.query.showThinking}`); - log.info(`Parameters from body: useAdvancedContext=${req.body?.useAdvancedContext}, showThinking=${req.body?.showThinking}, content=${content ? `${content.substring(0, 20)}...` : 'none'}`); + log.info(`LLM GET stream: chatNoteId=${req.params.chatNoteId}`); } - // Get chatNoteId from URL params chatNoteId = req.params.chatNoteId; - // For GET requests, ensure we have the stream parameter + // Validate inputs if (req.method === 'GET' && req.query.stream !== 'true') { throw new Error('Stream parameter must be set to true for GET/streaming requests'); } - // For POST requests, validate the content if (req.method === 'POST' && (!content || typeof content !== 'string' || content.trim().length === 0)) { throw new Error('Content cannot be empty'); } - // Get or create session from Chat Note - let session = await this.getOrCreateSessionFromChatNote(chatNoteId, req.method === 'POST'); + // Check if AI is enabled + const aiEnabled = await options.getOptionBool('aiEnabled'); + if (!aiEnabled) { + return { error: "AI features are disabled. Please enable them in the settings." }; + } - // If no session found and we're not allowed to create one (GET request) - if (!session && req.method === 'GET') { + if (!this.safelyUseAIManager()) { + return { error: "AI services are currently unavailable. Please check your configuration." }; + } + + // Load or create chat directly from storage + let chat = await chatStorageService.getChat(chatNoteId); + + if (!chat && req.method === 'GET') { throw new Error('Chat Note not found, cannot create session for streaming'); } - // For POST requests, if no Chat Note exists, create a new one - if (!session && req.method === 'POST') { - log.info(`No Chat Note found for ${chatNoteId}, creating a new Chat Note and session`); - - // Use the new Chat Note's ID for the session - session = SessionsStore.createSession({ - chatNoteId: chatNoteId - }); - - // Update the session ID to match the Chat Note ID - session.id = chatNoteId; - - log.info(`Created new Chat Note and session with ID: ${session.id}`); - - // Update the parameter to use the new ID - chatNoteId = session.id; + if (!chat && req.method === 'POST') { + log.info(`Creating new chat note with ID: ${chatNoteId}`); + chat = await chatStorageService.createChat('New Chat'); + // Update the chat ID to match the requested ID if possible + // In practice, we'll use the generated ID + chatNoteId = chat.id; } - // At this point, session should never be null - // TypeScript doesn't know this, so we'll add a check - if (!session) { - // This should never happen due to our logic above - throw new Error('Failed to create or retrieve session'); + if (!chat) { + throw new Error('Failed to create or retrieve chat'); } - // Update session last active timestamp - SessionsStore.touchSession(session.id); - - // For POST requests, store the user message - if (req.method === 'POST' && content && session) { - // Add message to session - session.messages.push({ + // For POST requests, add the user message + if (req.method === 'POST' && content) { + chat.messages.push({ role: 'user', - content, - timestamp: new Date() + content }); - - // Log a preview of the message log.info(`Processing LLM message: "${content.substring(0, 50)}${content.length > 50 ? '...' : ''}"`); } - // Check if AI services are enabled before proceeding - const aiEnabled = await options.getOptionBool('aiEnabled'); - log.info(`AI enabled setting: ${aiEnabled}`); - if (!aiEnabled) { - log.info("AI services are disabled by configuration"); - return { - error: "AI features are disabled. Please enable them in the settings." - }; - } - - // Check if AI services are available - log.info("Checking if AI services are available..."); - if (!this.safelyUseAIManager()) { - log.info("AI services are not available - checking for specific issues"); - - try { - // Create a direct instance to avoid circular references - const aiManager = new AIServiceManager(); - - if (!aiManager) { - log.error("AI service manager is not initialized"); - return { - error: "AI service is not properly initialized. Please check your configuration." - }; - } - - const availableProviders = aiManager.getAvailableProviders(); - if (availableProviders.length === 0) { - log.error("No AI providers are available"); - return { - error: "No AI providers are configured or available. Please check your AI settings." - }; - } - } catch (err) { - log.error(`Detailed AI service check failed: ${err}`); - } - - return { - error: "AI services are currently unavailable. Please check your configuration." - }; - } - - // Create direct instance to avoid circular references - const aiManager = new AIServiceManager(); - - // Get the default service - just use the first available one - const availableProviders = aiManager.getAvailableProviders(); - - if (availableProviders.length === 0) { - log.error("No AI providers are available after manager check"); - return { - error: "No AI providers are configured or available. Please check your AI settings." - }; - } - - // Use the first available provider - const providerName = availableProviders[0]; - log.info(`Using AI provider: ${providerName}`); - - // We know the manager has a 'services' property from our code inspection, - // but TypeScript doesn't know that from the interface. - // This is a workaround to access it - const service = (aiManager as any).services[providerName]; - - if (!service) { - log.error(`AI service for provider ${providerName} not found`); - return { - error: `Selected AI provider (${providerName}) is not available. Please check your configuration.` - }; - } - // Initialize tools - log.info("Initializing LLM agent tools..."); - // Ensure tools are initialized to prevent tool execution issues await ToolHandler.ensureToolsInitialized(); - // Create and use the chat pipeline instead of direct processing + // Create and use the chat pipeline const pipeline = new ChatPipeline({ enableStreaming: req.method === 'GET', enableMetrics: true, maxToolCallIterations: 5 }); - log.info("Executing chat pipeline..."); + // Get user's preferred model + const preferredModel = await this.getPreferredModel(); - // Create options object for better tracking const pipelineOptions = { - // Force useAdvancedContext to be a boolean, no matter what useAdvancedContext: useAdvancedContext === true, - systemPrompt: session?.messages.find(m => m.role === 'system')?.content, - temperature: session?.metadata.temperature, - maxTokens: session?.metadata.maxTokens, - // Get the user's preferred model if session model is 'default' or not set - model: await this.getPreferredModel(session?.metadata.model), - // Set stream based on request type, but ensure it's explicitly a boolean value - // GET requests or format=stream parameter indicates streaming should be used + systemPrompt: chat.messages.find(m => m.role === 'system')?.content, + model: preferredModel, stream: !!(req.method === 'GET' || req.query.format === 'stream' || req.query.stream === 'true'), - // Include chatNoteId for tracking tool executions chatNoteId: chatNoteId }; - // Log the options to verify what's being sent to the pipeline - log.info(`Pipeline input options: ${JSON.stringify({ - useAdvancedContext: pipelineOptions.useAdvancedContext, - stream: pipelineOptions.stream - })}`); + log.info(`Pipeline options: ${JSON.stringify({ useAdvancedContext: pipelineOptions.useAdvancedContext, stream: pipelineOptions.stream })}`); - // Import the WebSocket service for direct access + // Import WebSocket service for streaming const wsService = await import('../../ws.js'); + let accumulatedContent = ''; - // Create a stream callback wrapper - // This will ensure we properly handle all streaming messages - let messageContent = ''; - - // Prepare the pipeline input const pipelineInput: ChatPipelineInput = { - messages: session.messages.map(msg => ({ + messages: chat.messages.map(msg => ({ role: msg.role as 'user' | 'assistant' | 'system', content: msg.content })), - query: content || '', // Ensure query is always a string, even if content is null/undefined - noteId: session.noteContext ?? undefined, + query: content || '', + noteId: undefined, // TODO: Add context note support if needed showThinking: showThinking, options: pipelineOptions, streamCallback: req.method === 'GET' ? (data, done, rawChunk) => { - try { - // Use WebSocket service to send messages - this.handleStreamCallback( - data, done, rawChunk, - wsService.default, chatNoteId, - messageContent, session, res - ); - } catch (error) { - log.error(`Error in stream callback: ${error}`); - - // Try to send error message - try { - wsService.default.sendMessageToAllClients({ - type: 'llm-stream', - chatNoteId: chatNoteId, - error: `Stream error: ${error instanceof Error ? error.message : 'Unknown error'}`, - done: true - }); - - // End the response - res.write(`data: ${JSON.stringify({ error: 'Stream error', done: true })}\n\n`); - res.end(); - } catch (e) { - log.error(`Failed to send error message: ${e}`); - } - } + this.handleStreamCallback(data, done, rawChunk, wsService.default, chatNoteId, res); + if (data) accumulatedContent += data; } : undefined }; // Execute the pipeline const response = await pipeline.execute(pipelineInput); - // Handle the response if (req.method === 'POST') { - // Add assistant message to session - session.messages.push({ + // Add assistant response to chat + chat.messages.push({ role: 'assistant', - content: response.text || '', - timestamp: new Date() + content: response.text || '' }); - // Extract sources if they're available + // Save the updated chat back to storage (single source of truth) + await chatStorageService.updateChat(chat.id, chat.messages, chat.title); + + // Extract sources if available const sources = (response as any).sources || []; - // Store sources in the session metadata if they're present - if (sources.length > 0) { - session.metadata.sources = sources; - log.info(`Stored ${sources.length} sources in session metadata`); - } - - // Return the response with complete metadata return { content: response.text || '', sources: sources, metadata: { - model: response.model || session.metadata.model, - provider: response.provider || session.metadata.provider, - temperature: session.metadata.temperature, - maxTokens: session.metadata.maxTokens, - lastUpdated: new Date().toISOString(), - toolExecutions: session.metadata.toolExecutions || [] + model: response.model, + provider: response.provider, + lastUpdated: new Date().toISOString() } }; } else { - // For streaming requests, we've already sent the response + // For streaming, response is already sent via WebSocket/SSE + // Save the accumulated content + if (accumulatedContent) { + chat.messages.push({ + role: 'assistant', + content: accumulatedContent + }); + await chatStorageService.updateChat(chat.id, chat.messages, chat.title); + } return null; } - } catch (processingError: any) { - log.error(`Error processing message: ${processingError}`); - return { - error: `Error processing your request: ${processingError.message}` - }; + } catch (error: any) { + log.error(`Error processing message: ${error}`); + return { error: `Error processing your request: ${error.message}` }; } } /** - * Handle stream callback for WebSocket communication + * Simplified stream callback handler */ private handleStreamCallback( data: string | null, @@ -385,122 +219,72 @@ class RestChatService { rawChunk: any, wsService: any, chatNoteId: string, - messageContent: string, - session: any, res: Response ) { - // Only accumulate content that's actually text (not tool execution or thinking info) - if (data) { - messageContent += data; - } - - // Create a message object with all necessary fields const message: LLMStreamMessage = { type: 'llm-stream', - chatNoteId: chatNoteId + chatNoteId: chatNoteId, + done: done }; - // Add content if available - either the new chunk or full content on completion if (data) { message.content = data; } - // Add thinking info if available in the raw chunk if (rawChunk && 'thinking' in rawChunk && rawChunk.thinking) { message.thinking = rawChunk.thinking as string; } - // Add tool execution info if available in the raw chunk if (rawChunk && 'toolExecution' in rawChunk && rawChunk.toolExecution) { - // Transform the toolExecution to match the expected format const toolExec = rawChunk.toolExecution; message.toolExecution = { - // Use optional chaining for all properties - tool: typeof toolExec.tool === 'string' - ? toolExec.tool - : toolExec.tool?.name, + tool: typeof toolExec.tool === 'string' ? toolExec.tool : toolExec.tool?.name, result: toolExec.result, - // Map arguments to args args: 'arguments' in toolExec ? - (typeof toolExec.arguments === 'object' ? - toolExec.arguments as Record : {}) : {}, - // Add additional properties if they exist + (typeof toolExec.arguments === 'object' ? toolExec.arguments as Record : {}) : {}, action: 'action' in toolExec ? toolExec.action as string : undefined, toolCallId: 'toolCallId' in toolExec ? toolExec.toolCallId as string : undefined, error: 'error' in toolExec ? toolExec.error as string : undefined }; } - // Set done flag explicitly - message.done = done; - - // On final message, include the complete content too - if (done) { - // Store the response in the session when done - session.messages.push({ - role: 'assistant', - content: messageContent, - timestamp: new Date() - }); - } - - // Send message to all clients + // Send WebSocket message wsService.sendMessageToAllClients(message); - // Log what was sent (first message and completion) - if (message.thinking || done) { - log.info( - `[WS-SERVER] Sending LLM stream message: chatNoteId=${chatNoteId}, content=${!!message.content}, contentLength=${message.content?.length || 0}, thinking=${!!message.thinking}, toolExecution=${!!message.toolExecution}, done=${done}` - ); - } - - // For GET requests, also send as server-sent events - // Prepare response data for JSON event - const responseData: any = { - content: data, - done - }; - - // Add tool execution if available + // Send SSE response + const responseData: any = { content: data, done }; if (rawChunk?.toolExecution) { responseData.toolExecution = rawChunk.toolExecution; } - // Send the data as a JSON event res.write(`data: ${JSON.stringify(responseData)}\n\n`); - if (done) { res.end(); } } /** - * Create a new chat session + * Create a new chat */ async createSession(req: Request, res: Response) { try { const options: any = req.body || {}; const title = options.title || 'Chat Session'; - // Determine the note ID for the chat - let noteId = options.noteId || options.chatNoteId; // Accept either name for backward compatibility + let noteId = options.noteId || options.chatNoteId; - // If currentNoteId is provided, check if it's already an AI Chat note + // Check if currentNoteId is already an AI Chat note if (!noteId && options.currentNoteId) { - // Import becca to check note type const becca = (await import('../../../becca/becca.js')).default; const note = becca.notes[options.currentNoteId]; - // Check if this is an AI Chat note by looking at its content structure if (note) { try { const content = note.getContent(); if (content) { const contentStr = typeof content === 'string' ? content : content.toString(); const parsedContent = JSON.parse(contentStr); - // AI Chat notes have a messages array and noteId in their content - if (parsedContent.messages && Array.isArray(parsedContent.messages) && parsedContent.noteId) { - // This looks like an AI Chat note - use it directly + if (parsedContent.messages && Array.isArray(parsedContent.messages)) { noteId = options.currentNoteId; log.info(`Using existing AI Chat note ${noteId} as session`); } @@ -509,106 +293,69 @@ class RestChatService { // Not JSON content, so not an AI Chat note } } - - if (!noteId) { - log.info(`Creating new chat note from context of note ${options.currentNoteId}`); - // Don't use the currentNoteId as the chat note ID - create a new one - } } - // If we don't have a noteId, create a new Chat Note + // Create new chat if needed if (!noteId) { - // Create a new Chat Note via the storage service - const chatStorageService = (await import('../../llm/chat_storage_service.js')).default; const newChat = await chatStorageService.createChat(title); noteId = newChat.id; log.info(`Created new Chat Note with ID: ${noteId}`); } else { - // We have a noteId - this means we're working with an existing aiChat note - // Don't create another note, just use the existing one log.info(`Using existing Chat Note with ID: ${noteId}`); } - // Create a new session through our session store using the note ID - const session = SessionsStore.createSession({ - chatNoteId: noteId, // This is really the noteId of the chat note - title, - systemPrompt: options.systemPrompt, - contextNoteId: options.contextNoteId, - maxTokens: options.maxTokens, - model: options.model, - provider: options.provider, - temperature: options.temperature - }); - return { - id: session.id, // This will be the same as noteId - title: session.title, - createdAt: session.createdAt, - noteId: noteId // Return the note ID for clarity + id: noteId, + title: title, + createdAt: new Date(), + noteId: noteId }; } catch (error: any) { - log.error(`Error creating LLM session: ${error.message || 'Unknown error'}`); - throw new Error(`Failed to create LLM session: ${error.message || 'Unknown error'}`); + log.error(`Error creating chat session: ${error.message || 'Unknown error'}`); + throw new Error(`Failed to create chat session: ${error.message || 'Unknown error'}`); } } /** - * Get a specific chat session by ID + * Get a chat by ID */ - async getSession(req: Request, res: Response) { + async getSession(req: Request, res: Response): Promise { try { const { sessionId } = req.params; - // Check if session exists - const session = SessionsStore.getSession(sessionId); - if (!session) { - // Instead of throwing an error, return a structured 404 response - // that the frontend can handle gracefully + const chat = await chatStorageService.getChat(sessionId); + if (!chat) { res.status(404).json({ error: true, message: `Session with ID ${sessionId} not found`, code: 'session_not_found', sessionId }); - return null; // Return null to prevent further processing + return null; } - // Return session with metadata and additional fields return { - id: session.id, - title: session.title, - createdAt: session.createdAt, - lastActive: session.lastActive, - messages: session.messages, - noteContext: session.noteContext, - // Include additional fields for the frontend - sources: session.metadata.sources || [], - metadata: { - model: session.metadata.model, - provider: session.metadata.provider, - temperature: session.metadata.temperature, - maxTokens: session.metadata.maxTokens, - lastUpdated: session.lastActive.toISOString(), - // Include simplified tool executions if available - toolExecutions: session.metadata.toolExecutions || [] - } + id: chat.id, + title: chat.title, + createdAt: chat.createdAt, + lastActive: chat.updatedAt, + messages: chat.messages, + metadata: chat.metadata || {} }; } catch (error: any) { - log.error(`Error getting LLM session: ${error.message || 'Unknown error'}`); + log.error(`Error getting chat session: ${error.message || 'Unknown error'}`); throw new Error(`Failed to get session: ${error.message || 'Unknown error'}`); } } /** - * Delete a chat session + * Delete a chat */ async deleteSession(req: Request, res: Response) { try { const { sessionId } = req.params; - // Delete the session - const success = SessionsStore.deleteSession(sessionId); + const success = await chatStorageService.deleteChat(sessionId); if (!success) { throw new Error(`Session with ID ${sessionId} not found`); } @@ -618,116 +365,46 @@ class RestChatService { message: `Session ${sessionId} deleted successfully` }; } catch (error: any) { - log.error(`Error deleting LLM session: ${error.message || 'Unknown error'}`); + log.error(`Error deleting chat session: ${error.message || 'Unknown error'}`); throw new Error(`Failed to delete session: ${error.message || 'Unknown error'}`); } } /** - * Get all sessions + * Get all chats */ - getSessions() { - return SessionsStore.getAllSessions(); - } - - /** - * Create an in-memory session from a Chat Note - * This treats the Chat Note as the source of truth, using its ID as the session ID - */ - async createSessionFromChatNote(noteId: string): Promise { + async getAllSessions() { try { - log.info(`Creating in-memory session for Chat Note ID ${noteId}`); - - // Import chat storage service - const chatStorageService = (await import('../../llm/chat_storage_service.js')).default; - - // Try to get the Chat Note data - const chatNote = await chatStorageService.getChat(noteId); - - if (!chatNote) { - log.error(`Chat Note ${noteId} not found, cannot create session`); - return null; - } - - log.info(`Found Chat Note ${noteId}, creating in-memory session`); - - // Convert Message[] to ChatMessage[] by ensuring the role is compatible - const chatMessages: ChatMessage[] = chatNote.messages.map(msg => ({ - role: msg.role === 'tool' ? 'assistant' : msg.role, // Map 'tool' role to 'assistant' - content: msg.content, - timestamp: new Date() - })); - - // Create a new session with the same ID as the Chat Note - const session: ChatSession = { - id: chatNote.id, // Use Chat Note ID as the session ID - title: chatNote.title, - messages: chatMessages, - createdAt: chatNote.createdAt || new Date(), - lastActive: new Date(), - metadata: chatNote.metadata || {} + const chats = await chatStorageService.getAllChats(); + return { + sessions: chats.map(chat => ({ + id: chat.id, + title: chat.title, + createdAt: chat.createdAt, + lastActive: chat.updatedAt, + messageCount: chat.messages.length + })) }; - - // Add the session to the in-memory store - SessionsStore.getAllSessions().set(noteId, session); - - log.info(`Successfully created in-memory session for Chat Note ${noteId}`); - return session; - } catch (error) { - log.error(`Failed to create session from Chat Note: ${error}`); - return null; + } catch (error: any) { + log.error(`Error listing sessions: ${error}`); + throw new Error(`Failed to list sessions: ${error}`); } } /** - * Get an existing session or create a new one from a Chat Note - * This treats the Chat Note as the source of truth, using its ID as the session ID + * Get the user's preferred model */ - async getOrCreateSessionFromChatNote(noteId: string, createIfNotFound: boolean = true): Promise { - // First check if we already have this session in memory - let session = SessionsStore.getSession(noteId); - - if (session) { - log.info(`Found existing in-memory session for Chat Note ${noteId}`); - return session; - } - - // If not in memory, try to create from Chat Note - log.info(`Session not found in memory for Chat Note ${noteId}, attempting to create it`); - - // Only try to create if allowed - if (!createIfNotFound) { - log.info(`Not creating new session for ${noteId} as createIfNotFound=false`); - return null; - } - - // Create from Chat Note - return await this.createSessionFromChatNote(noteId); - } - - /** - * Get the user's preferred model using the new configuration system - */ - async getPreferredModel(sessionModel?: string): Promise { - // If the session already has a valid model (not 'default'), use it - if (sessionModel && sessionModel !== 'default') { - return sessionModel; - } - + async getPreferredModel(): Promise { try { - // Use the new configuration system - no string parsing! const validConfig = await getFirstValidModelConfig(); - if (!validConfig) { - log.error('No valid AI model configuration found. Please configure your AI settings.'); - return undefined; // Don't provide fallback defaults + log.error('No valid AI model configuration found'); + return undefined; } - - log.info(`Selected user's preferred model: ${validConfig.model} from provider: ${validConfig.provider}`); return validConfig.model; } catch (error) { - log.error(`Error getting user's preferred model: ${error}`); - return undefined; // Don't provide fallback defaults, let the caller handle it + log.error(`Error getting preferred model: ${error}`); + return undefined; } } } diff --git a/apps/server/src/services/llm/chat/sessions_store.ts b/apps/server/src/services/llm/chat/sessions_store.ts deleted file mode 100644 index c16b1af04..000000000 --- a/apps/server/src/services/llm/chat/sessions_store.ts +++ /dev/null @@ -1,168 +0,0 @@ -/** - * In-memory storage for chat sessions - */ -import log from "../../log.js"; -import { LLM_CONSTANTS } from '../constants/provider_constants.js'; -import { SEARCH_CONSTANTS } from '../constants/search_constants.js'; -import type { ChatSession, ChatMessage } from '../interfaces/chat_session.js'; - -// In-memory storage for sessions -const sessions = new Map(); - -// Flag to track if cleanup timer has been initialized -let cleanupInitialized = false; - -/** - * Provides methods to manage chat sessions - */ -class SessionsStore { - /** - * Initialize the session cleanup timer to remove old/inactive sessions - */ - initializeCleanupTimer(): void { - if (cleanupInitialized) { - return; - } - - // Clean sessions that have expired based on the constants - function cleanupOldSessions() { - const expiryTime = new Date(Date.now() - LLM_CONSTANTS.SESSION.SESSION_EXPIRY_MS); - for (const [sessionId, session] of sessions.entries()) { - if (session.lastActive < expiryTime) { - sessions.delete(sessionId); - } - } - } - - // Run cleanup at the configured interval - setInterval(cleanupOldSessions, LLM_CONSTANTS.SESSION.CLEANUP_INTERVAL_MS); - cleanupInitialized = true; - log.info("Session cleanup timer initialized"); - } - - /** - * Get all sessions - */ - getAllSessions(): Map { - return sessions; - } - - /** - * Get a specific session by ID - */ - getSession(sessionId: string): ChatSession | undefined { - return sessions.get(sessionId); - } - - /** - * Create a new session - */ - createSession(options: { - chatNoteId: string; - title?: string; - systemPrompt?: string; - contextNoteId?: string; - maxTokens?: number; - model?: string; - provider?: string; - temperature?: number; - }): ChatSession { - this.initializeCleanupTimer(); - - const title = options.title || 'Chat Session'; - const sessionId = options.chatNoteId; - const now = new Date(); - - // Initial system message if provided - const messages: ChatMessage[] = []; - if (options.systemPrompt) { - messages.push({ - role: 'system', - content: options.systemPrompt, - timestamp: now - }); - } - - // Create and store the session - const session: ChatSession = { - id: sessionId, - title, - messages, - createdAt: now, - lastActive: now, - noteContext: options.contextNoteId, - metadata: { - temperature: options.temperature || SEARCH_CONSTANTS.TEMPERATURE.DEFAULT, - maxTokens: options.maxTokens, - model: options.model, - provider: options.provider, - sources: [], - toolExecutions: [], - lastUpdated: now.toISOString() - } - }; - - sessions.set(sessionId, session); - log.info(`Created in-memory session for Chat Note ID: ${sessionId}`); - - return session; - } - - /** - * Update a session's last active timestamp - */ - touchSession(sessionId: string): boolean { - const session = sessions.get(sessionId); - if (!session) { - return false; - } - - session.lastActive = new Date(); - return true; - } - - /** - * Delete a session - */ - deleteSession(sessionId: string): boolean { - return sessions.delete(sessionId); - } - - /** - * Record a tool execution in the session metadata - */ - recordToolExecution(chatNoteId: string, tool: any, result: string, error?: string): void { - if (!chatNoteId) return; - - const session = sessions.get(chatNoteId); - if (!session) return; - - try { - const toolExecutions = session.metadata.toolExecutions || []; - - // Format tool execution record - const execution = { - id: tool.id || `tool-${Date.now()}-${Math.random().toString(36).substring(2, 7)}`, - name: tool.function?.name || 'unknown', - arguments: typeof tool.function?.arguments === 'string' - ? (() => { try { return JSON.parse(tool.function.arguments); } catch { return tool.function.arguments; } })() - : tool.function?.arguments || {}, - result: result, - error: error, - timestamp: new Date().toISOString() - }; - - // Add to tool executions - toolExecutions.push(execution); - session.metadata.toolExecutions = toolExecutions; - - log.info(`Recorded tool execution for ${execution.name} in session ${chatNoteId}`); - } catch (err) { - log.error(`Failed to record tool execution: ${err}`); - } - } -} - -// Create singleton instance -const sessionsStore = new SessionsStore(); -export default sessionsStore; From 7b498cf384d1ffd389e62dda1967ace746316670 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Mon, 2 Jun 2025 22:30:59 +0000 Subject: [PATCH 10/18] refactor(llm): update chat saving logic to prevent race conditions between client and server --- .../src/widgets/llm_chat/llm_chat_panel.ts | 48 ++++++++++++++----- .../services/llm/chat/rest_chat_service.ts | 25 +++++++--- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/apps/client/src/widgets/llm_chat/llm_chat_panel.ts b/apps/client/src/widgets/llm_chat/llm_chat_panel.ts index fc6a28f26..1350302ac 100644 --- a/apps/client/src/widgets/llm_chat/llm_chat_panel.ts +++ b/apps/client/src/widgets/llm_chat/llm_chat_panel.ts @@ -951,9 +951,9 @@ export default class LlmChatPanel extends BasicWidget { } } - // Save the updated data to the note - this.saveCurrentData() - .catch(err => console.error("Failed to save data after streaming completed:", err)); + // DON'T save here - let the server handle saving the complete conversation + // to avoid race conditions between client and server saves + console.log("Updated metadata after streaming completion, server should save"); }) .catch(err => console.error("Error fetching session data after streaming:", err)); } @@ -991,11 +991,9 @@ export default class LlmChatPanel extends BasicWidget { console.log(`Cached tool execution for ${toolData.tool} to be saved later`); - // Save immediately after receiving a tool execution - // This ensures we don't lose tool execution data if streaming fails - this.saveCurrentData().catch(err => { - console.error("Failed to save tool execution data:", err); - }); + // DON'T save immediately during streaming - let the server handle saving + // to avoid race conditions between client and server saves + console.log(`Tool execution cached, will be saved by server`); } }, // Complete handler @@ -1078,10 +1076,36 @@ export default class LlmChatPanel extends BasicWidget { // Hide loading indicator hideLoadingIndicator(this.loadingIndicator); - // Save the final state to the Chat Note - this.saveCurrentData().catch(err => { - console.error("Failed to save assistant response to note:", err); - }); + // DON'T save here immediately - let the server save the accumulated response first + // to avoid race conditions. We'll reload the data from the server after a short delay. + console.log("Stream completed, waiting for server to save then reloading data..."); + setTimeout(async () => { + try { + console.log("About to reload data from server..."); + const currentMessageCount = this.messages.length; + console.log(`Current client message count before reload: ${currentMessageCount}`); + + // Reload the data from the server which should have the complete conversation + const reloadSuccess = await this.loadSavedData(); + + const newMessageCount = this.messages.length; + console.log(`Reload success: ${reloadSuccess}, message count after reload: ${newMessageCount}`); + + if (reloadSuccess && newMessageCount > currentMessageCount) { + console.log("Successfully reloaded data with more complete conversation"); + } else if (!reloadSuccess) { + console.warn("Reload failed, keeping current client state"); + } else { + console.warn("Reload succeeded but message count didn't increase"); + } + } catch (error) { + console.error("Failed to reload data after stream completion:", error); + // Fallback: save our current state if reload fails + this.saveCurrentData().catch(err => { + console.error("Failed to save assistant response to note:", err); + }); + } + }, 3000); // Wait 3 seconds for server to complete its save } // Scroll to bottom diff --git a/apps/server/src/services/llm/chat/rest_chat_service.ts b/apps/server/src/services/llm/chat/rest_chat_service.ts index b6b0bc820..53c682862 100644 --- a/apps/server/src/services/llm/chat/rest_chat_service.ts +++ b/apps/server/src/services/llm/chat/rest_chat_service.ts @@ -150,7 +150,7 @@ class RestChatService { // Import WebSocket service for streaming const wsService = await import('../../ws.js'); - let accumulatedContent = ''; + const accumulatedContentRef = { value: '' }; const pipelineInput: ChatPipelineInput = { messages: chat.messages.map(msg => ({ @@ -162,8 +162,7 @@ class RestChatService { showThinking: showThinking, options: pipelineOptions, streamCallback: req.method === 'GET' ? (data, done, rawChunk) => { - this.handleStreamCallback(data, done, rawChunk, wsService.default, chatNoteId, res); - if (data) accumulatedContent += data; + this.handleStreamCallback(data, done, rawChunk, wsService.default, chatNoteId, res, accumulatedContentRef); } : undefined }; @@ -194,13 +193,15 @@ class RestChatService { }; } else { // For streaming, response is already sent via WebSocket/SSE - // Save the accumulated content - if (accumulatedContent) { + // Save the accumulated content - prefer accumulated content over response.text + const finalContent = accumulatedContentRef.value || response.text || ''; + if (finalContent) { chat.messages.push({ role: 'assistant', - content: accumulatedContent + content: finalContent }); await chatStorageService.updateChat(chat.id, chat.messages, chat.title); + log.info(`Saved accumulated streaming content: ${finalContent.length} characters`); } return null; } @@ -219,7 +220,8 @@ class RestChatService { rawChunk: any, wsService: any, chatNoteId: string, - res: Response + res: Response, + accumulatedContentRef: { value: string } ) { const message: LLMStreamMessage = { type: 'llm-stream', @@ -229,6 +231,15 @@ class RestChatService { if (data) { message.content = data; + // Handle accumulation carefully - if this appears to be a complete response + // (done=true and data is much longer than current accumulated), replace rather than append + if (done && data.length > accumulatedContentRef.value.length && data.includes(accumulatedContentRef.value)) { + // This looks like a complete final response that includes what we've accumulated + accumulatedContentRef.value = data; + } else { + // Normal incremental accumulation + accumulatedContentRef.value += data; + } } if (rawChunk && 'thinking' in rawChunk && rawChunk.thinking) { From 0d305cd22cda2d51651d2ab8cf3d31581cf9ac7d Mon Sep 17 00:00:00 2001 From: perf3ct Date: Mon, 2 Jun 2025 22:34:10 +0000 Subject: [PATCH 11/18] refactor(llm): optimize chat UI updates by checking for new content and preserving scroll position --- .../src/widgets/llm_chat/llm_chat_panel.ts | 71 +++++++++++++++++-- 1 file changed, 64 insertions(+), 7 deletions(-) diff --git a/apps/client/src/widgets/llm_chat/llm_chat_panel.ts b/apps/client/src/widgets/llm_chat/llm_chat_panel.ts index 1350302ac..b9b3228ef 100644 --- a/apps/client/src/widgets/llm_chat/llm_chat_panel.ts +++ b/apps/client/src/widgets/llm_chat/llm_chat_panel.ts @@ -362,16 +362,52 @@ export default class LlmChatPanel extends BasicWidget { const savedData = await this.onGetData() as ChatData; if (savedData?.messages?.length > 0) { + // Check if we actually have new content to avoid unnecessary UI rebuilds + const currentMessageCount = this.messages.length; + const savedMessageCount = savedData.messages.length; + + // If message counts are the same, check if content is different + const hasNewContent = savedMessageCount > currentMessageCount || + JSON.stringify(this.messages) !== JSON.stringify(savedData.messages); + + if (!hasNewContent) { + console.log("No new content detected, skipping UI rebuild"); + return true; + } + + console.log(`Loading saved data: ${currentMessageCount} -> ${savedMessageCount} messages`); + + // Store current scroll position if we need to preserve it + const shouldPreserveScroll = savedMessageCount > currentMessageCount && currentMessageCount > 0; + const currentScrollTop = shouldPreserveScroll ? this.chatContainer.scrollTop : 0; + const currentScrollHeight = shouldPreserveScroll ? this.chatContainer.scrollHeight : 0; + // Load messages + const oldMessages = [...this.messages]; this.messages = savedData.messages; - // Clear and rebuild the chat UI - this.noteContextChatMessages.innerHTML = ''; + // Only rebuild UI if we have significantly different content + if (savedMessageCount > currentMessageCount) { + // We have new messages - just add the new ones instead of rebuilding everything + const newMessages = savedData.messages.slice(currentMessageCount); + console.log(`Adding ${newMessages.length} new messages to UI`); - this.messages.forEach(message => { - const role = message.role as 'user' | 'assistant'; - this.addMessageToChat(role, message.content); - }); + newMessages.forEach(message => { + const role = message.role as 'user' | 'assistant'; + this.addMessageToChat(role, message.content); + }); + } else { + // Content changed but count is same - need to rebuild + console.log("Message content changed, rebuilding UI"); + + // Clear and rebuild the chat UI + this.noteContextChatMessages.innerHTML = ''; + + this.messages.forEach(message => { + const role = message.role as 'user' | 'assistant'; + this.addMessageToChat(role, message.content); + }); + } // Restore tool execution steps if they exist if (savedData.toolSteps && Array.isArray(savedData.toolSteps) && savedData.toolSteps.length > 0) { @@ -421,6 +457,27 @@ export default class LlmChatPanel extends BasicWidget { return false; } + // Restore scroll position if we were preserving it + if (shouldPreserveScroll) { + // Calculate the new scroll position to maintain relative position + const newScrollHeight = this.chatContainer.scrollHeight; + const scrollDifference = newScrollHeight - currentScrollHeight; + const newScrollTop = currentScrollTop + scrollDifference; + + // Only scroll down if we're near the bottom, otherwise preserve exact position + const wasNearBottom = (currentScrollTop + this.chatContainer.clientHeight) >= (currentScrollHeight - 50); + + if (wasNearBottom) { + // User was at bottom, scroll to new bottom + this.chatContainer.scrollTop = newScrollHeight; + console.log("User was at bottom, scrolling to new bottom"); + } else { + // User was not at bottom, try to preserve their position + this.chatContainer.scrollTop = newScrollTop; + console.log(`Preserving scroll position: ${currentScrollTop} -> ${newScrollTop}`); + } + } + return true; } } catch (error) { @@ -1105,7 +1162,7 @@ export default class LlmChatPanel extends BasicWidget { console.error("Failed to save assistant response to note:", err); }); } - }, 3000); // Wait 3 seconds for server to complete its save + }, 1500); // Wait 1.5 seconds for server to complete its save } // Scroll to bottom From aad92b57c783f571c15a8fc9cdf90fcdd72db91c Mon Sep 17 00:00:00 2001 From: perf3ct Date: Mon, 2 Jun 2025 22:47:30 +0000 Subject: [PATCH 12/18] fix(llm): prevent sent message duplication --- apps/server/src/routes/api/llm.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/apps/server/src/routes/api/llm.ts b/apps/server/src/routes/api/llm.ts index 80bc36884..05e8ee879 100644 --- a/apps/server/src/routes/api/llm.ts +++ b/apps/server/src/routes/api/llm.ts @@ -858,15 +858,6 @@ async function streamMessage(req: Request, res: Response) { } } - // Add user message to the chat (without timestamp since Message interface doesn't support it) - chat.messages.push({ - role: 'user', - content: enhancedContent - }); - - // Save the updated chat - await chatStorageService.updateChat(chat.id, chat.messages, chat.title); - // Create request parameters for the pipeline const requestParams = { chatNoteId: chatNoteId, From e7e04b7ccda12fb4a64f8f74ffeaeb2a2c0f6bb4 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Mon, 2 Jun 2025 23:25:15 +0000 Subject: [PATCH 13/18] refactor(llm): streamline chat response handling by simplifying content accumulation and removing unnecessary thinking content processing --- .../src/widgets/llm_chat/communication.ts | 282 +++--------------- .../src/widgets/llm_chat/llm_chat_panel.ts | 70 +---- apps/server/src/routes/api/llm.ts | 118 +++----- .../services/llm/chat/rest_chat_service.ts | 15 +- 4 files changed, 87 insertions(+), 398 deletions(-) diff --git a/apps/client/src/widgets/llm_chat/communication.ts b/apps/client/src/widgets/llm_chat/communication.ts index 87687cab9..614add7ad 100644 --- a/apps/client/src/widgets/llm_chat/communication.ts +++ b/apps/client/src/widgets/llm_chat/communication.ts @@ -62,15 +62,11 @@ export async function setupStreamingResponse( ): Promise { return new Promise((resolve, reject) => { let assistantResponse = ''; - let postToolResponse = ''; // Separate accumulator for post-tool execution content let receivedAnyContent = false; - let receivedPostToolContent = false; // Track if we've started receiving post-tool content let timeoutId: number | null = null; let initialTimeoutId: number | null = null; let cleanupTimeoutId: number | null = null; let receivedAnyMessage = false; - let toolsExecuted = false; // Flag to track if tools were executed in this session - let toolExecutionCompleted = false; // Flag to track if tool execution is completed let eventListener: ((event: Event) => void) | null = null; let lastMessageTimestamp = 0; @@ -118,28 +114,14 @@ export async function setupStreamingResponse( resolve(); }; - // Function to schedule cleanup with ability to cancel - const scheduleCleanup = (delay: number) => { - // Clear any existing cleanup timeout - if (cleanupTimeoutId) { - window.clearTimeout(cleanupTimeoutId); + // Set initial timeout to catch cases where no message is received at all + initialTimeoutId = window.setTimeout(() => { + if (!receivedAnyMessage) { + console.error(`[${responseId}] No initial message received within timeout`); + performCleanup(); + reject(new Error('No response received from server')); } - - console.log(`[${responseId}] Scheduling listener cleanup in ${delay}ms`); - - // Set new cleanup timeout - cleanupTimeoutId = window.setTimeout(() => { - // Only clean up if no messages received recently (in last 2 seconds) - const timeSinceLastMessage = Date.now() - lastMessageTimestamp; - if (timeSinceLastMessage > 2000) { - performCleanup(); - } else { - console.log(`[${responseId}] Received message recently, delaying cleanup`); - // Reschedule cleanup - scheduleCleanup(2000); - } - }, delay); - }; + }, 10000); // Create a message handler for CustomEvents eventListener = (event: Event) => { @@ -161,7 +143,7 @@ export async function setupStreamingResponse( cleanupTimeoutId = null; } - console.log(`[${responseId}] LLM Stream message received via CustomEvent: chatNoteId=${noteId}, content=${!!message.content}, contentLength=${message.content?.length || 0}, thinking=${!!message.thinking}, toolExecution=${!!message.toolExecution}, done=${!!message.done}, type=${message.type || 'llm-stream'}`); + console.log(`[${responseId}] LLM Stream message received: content=${!!message.content}, contentLength=${message.content?.length || 0}, thinking=${!!message.thinking}, toolExecution=${!!message.toolExecution}, done=${!!message.done}`); // Mark first message received if (!receivedAnyMessage) { @@ -175,109 +157,33 @@ export async function setupStreamingResponse( } } - // Handle specific message types - if (message.type === 'tool_execution_start') { - toolsExecuted = true; // Mark that tools were executed - onThinkingUpdate('Executing tools...'); - // Also trigger tool execution UI with a specific format - onToolExecution({ - action: 'start', - tool: 'tools', - result: 'Executing tools...' - }); - return; // Skip accumulating content from this message + // Handle error + if (message.error) { + console.error(`[${responseId}] Stream error: ${message.error}`); + performCleanup(); + reject(new Error(message.error)); + return; } - if (message.type === 'tool_result' && message.toolExecution) { - toolsExecuted = true; // Mark that tools were executed - console.log(`[${responseId}] Processing tool result: ${JSON.stringify(message.toolExecution)}`); + // Handle thinking updates - only show if showThinking is enabled + if (message.thinking && messageParams.showThinking) { + console.log(`[${responseId}] Received thinking: ${message.thinking.substring(0, 100)}...`); + onThinkingUpdate(message.thinking); + } - // If tool execution doesn't have an action, add 'result' as the default - if (!message.toolExecution.action) { - message.toolExecution.action = 'result'; - } - - // First send a 'start' action to ensure the container is created - onToolExecution({ - action: 'start', - tool: 'tools', - result: 'Tool execution initialized' - }); - - // Then send the actual tool execution data + // Handle tool execution updates + if (message.toolExecution) { + console.log(`[${responseId}] Tool execution update:`, message.toolExecution); onToolExecution(message.toolExecution); - - // Mark tool execution as completed if this is a result or error - if (message.toolExecution.action === 'result' || message.toolExecution.action === 'complete' || message.toolExecution.action === 'error') { - toolExecutionCompleted = true; - console.log(`[${responseId}] Tool execution completed`); - } - - return; // Skip accumulating content from this message - } - - if (message.type === 'tool_execution_error' && message.toolExecution) { - toolsExecuted = true; // Mark that tools were executed - toolExecutionCompleted = true; // Mark tool execution as completed - onToolExecution({ - ...message.toolExecution, - action: 'error', - error: message.toolExecution.error || 'Unknown error during tool execution' - }); - return; // Skip accumulating content from this message - } - - if (message.type === 'tool_completion_processing') { - toolsExecuted = true; // Mark that tools were executed - toolExecutionCompleted = true; // Tools are done, now processing the result - onThinkingUpdate('Generating response with tool results...'); - // Also trigger tool execution UI with a specific format - onToolExecution({ - action: 'generating', - tool: 'tools', - result: 'Generating response with tool results...' - }); - return; // Skip accumulating content from this message } // Handle content updates if (message.content) { - console.log(`[${responseId}] Received content chunk of length ${message.content.length}, preview: "${message.content.substring(0, 50)}${message.content.length > 50 ? '...' : ''}"`); - - // If tools were executed and completed, and we're now getting new content, - // this is likely the final response after tool execution from Anthropic - if (toolsExecuted && toolExecutionCompleted && message.content) { - console.log(`[${responseId}] Post-tool execution content detected`); - - // If this is the first post-tool chunk, indicate we're starting a new response - if (!receivedPostToolContent) { - receivedPostToolContent = true; - postToolResponse = ''; // Clear any previous post-tool response - console.log(`[${responseId}] First post-tool content chunk, starting fresh accumulation`); - } - - // Accumulate post-tool execution content - postToolResponse += message.content; - console.log(`[${responseId}] Accumulated post-tool content, now ${postToolResponse.length} chars`); - - // Update the UI with the accumulated post-tool content - // This replaces the pre-tool content with our accumulated post-tool content - onContentUpdate(postToolResponse, message.done || false); - } else { - // Standard content handling for non-tool cases or initial tool response - - // Check if this is a duplicated message containing the same content we already have - if (message.done && assistantResponse.includes(message.content)) { - console.log(`[${responseId}] Ignoring duplicated content in done message`); - } else { - // Add to our accumulated response - assistantResponse += message.content; - } - - // Update the UI immediately with each chunk - onContentUpdate(assistantResponse, message.done || false); - } + // Simply append the new content - no complex deduplication + assistantResponse += message.content; + // Update the UI immediately with each chunk + onContentUpdate(assistantResponse, message.done || false); receivedAnyContent = true; // Reset timeout since we got content @@ -288,150 +194,32 @@ export async function setupStreamingResponse( // Set new timeout timeoutId = window.setTimeout(() => { console.warn(`[${responseId}] Stream timeout for chat note ${noteId}`); - - // Clean up performCleanup(); reject(new Error('Stream timeout')); }, 30000); } - // Handle tool execution updates (legacy format and standard format with llm-stream type) - if (message.toolExecution) { - // Only process if we haven't already handled this message via specific message types - if (message.type === 'llm-stream' || !message.type) { - console.log(`[${responseId}] Received tool execution update: action=${message.toolExecution.action || 'unknown'}`); - toolsExecuted = true; // Mark that tools were executed - - // Mark tool execution as completed if this is a result or error - if (message.toolExecution.action === 'result' || - message.toolExecution.action === 'complete' || - message.toolExecution.action === 'error') { - toolExecutionCompleted = true; - console.log(`[${responseId}] Tool execution completed via toolExecution message`); - } - - onToolExecution(message.toolExecution); - } - } - - // Handle tool calls from the raw data or direct in message (OpenAI format) - const toolCalls = message.tool_calls || (message.raw && message.raw.tool_calls); - if (toolCalls && Array.isArray(toolCalls)) { - console.log(`[${responseId}] Received tool calls: ${toolCalls.length} tools`); - toolsExecuted = true; // Mark that tools were executed - - // First send a 'start' action to ensure the container is created - onToolExecution({ - action: 'start', - tool: 'tools', - result: 'Tool execution initialized' - }); - - // Then process each tool call - for (const toolCall of toolCalls) { - let args = toolCall.function?.arguments || {}; - - // Try to parse arguments if they're a string - if (typeof args === 'string') { - try { - args = JSON.parse(args); - } catch (e) { - console.log(`[${responseId}] Could not parse tool arguments as JSON: ${e}`); - args = { raw: args }; - } - } - - onToolExecution({ - action: 'executing', - tool: toolCall.function?.name || 'unknown', - toolCallId: toolCall.id, - args: args - }); - } - } - - // Handle thinking state updates - if (message.thinking) { - console.log(`[${responseId}] Received thinking update: ${message.thinking.substring(0, 50)}...`); - onThinkingUpdate(message.thinking); - } - // Handle completion if (message.done) { - console.log(`[${responseId}] Stream completed for chat note ${noteId}, has content: ${!!message.content}, content length: ${message.content?.length || 0}, current response: ${assistantResponse.length} chars`); + console.log(`[${responseId}] Stream completed for chat note ${noteId}, final response: ${assistantResponse.length} chars`); - // Dump message content to console for debugging - if (message.content) { - console.log(`[${responseId}] CONTENT IN DONE MESSAGE (first 200 chars): "${message.content.substring(0, 200)}..."`); - - // Check if the done message contains the exact same content as our accumulated response - // We normalize by removing whitespace to avoid false negatives due to spacing differences - const normalizedMessage = message.content.trim(); - const normalizedResponse = assistantResponse.trim(); - - if (normalizedMessage === normalizedResponse) { - console.log(`[${responseId}] Final message is identical to accumulated response, no need to update`); - } - // If the done message is longer but contains our accumulated response, use the done message - else if (normalizedMessage.includes(normalizedResponse) && normalizedMessage.length > normalizedResponse.length) { - console.log(`[${responseId}] Final message is more complete than accumulated response, using it`); - assistantResponse = message.content; - } - // If the done message is different and not already included, append it to avoid duplication - else if (!normalizedResponse.includes(normalizedMessage) && normalizedMessage.length > 0) { - console.log(`[${responseId}] Final message has unique content, using it`); - assistantResponse = message.content; - } - // Otherwise, we already have the content accumulated, so no need to update - else { - console.log(`[${responseId}] Already have this content accumulated, not updating`); - } - } - - // Clear timeout if set + // Clear all timeouts if (timeoutId !== null) { window.clearTimeout(timeoutId); timeoutId = null; } - // Always mark as done when we receive the done flag - onContentUpdate(assistantResponse, true); - - // Set a longer delay before cleanup to allow for post-tool execution messages - // Especially important for Anthropic which may send final message after tool execution - const cleanupDelay = toolsExecuted ? 15000 : 1000; // 15 seconds if tools were used, otherwise 1 second - console.log(`[${responseId}] Setting cleanup delay of ${cleanupDelay}ms since toolsExecuted=${toolsExecuted}`); - scheduleCleanup(cleanupDelay); + // Schedule cleanup after a brief delay to ensure all processing is complete + cleanupTimeoutId = window.setTimeout(() => { + performCleanup(); + }, 100); } }; - // Register event listener for the custom event - try { - window.addEventListener('llm-stream-message', eventListener); - console.log(`[${responseId}] Event listener added for llm-stream-message events`); - } catch (err) { - console.error(`[${responseId}] Error setting up event listener:`, err); - reject(err); - return; - } + // Register the event listener for WebSocket messages + window.addEventListener('llm-stream-message', eventListener); - // Set initial timeout for receiving any message - initialTimeoutId = window.setTimeout(() => { - console.warn(`[${responseId}] No messages received for initial period in chat note ${noteId}`); - if (!receivedAnyMessage) { - console.error(`[${responseId}] WebSocket connection not established for chat note ${noteId}`); - - if (timeoutId !== null) { - window.clearTimeout(timeoutId); - } - - // Clean up - cleanupEventListener(eventListener); - - // Show error message to user - reject(new Error('WebSocket connection not established')); - } - }, 10000); + console.log(`[${responseId}] Event listener registered, waiting for messages...`); }); } diff --git a/apps/client/src/widgets/llm_chat/llm_chat_panel.ts b/apps/client/src/widgets/llm_chat/llm_chat_panel.ts index b9b3228ef..ce9b5dd12 100644 --- a/apps/client/src/widgets/llm_chat/llm_chat_panel.ts +++ b/apps/client/src/widgets/llm_chat/llm_chat_panel.ts @@ -1068,16 +1068,6 @@ export default class LlmChatPanel extends BasicWidget { * Update the UI with streaming content */ private updateStreamingUI(assistantResponse: string, isDone: boolean = false) { - // Parse and handle thinking content if present - if (!isDone) { - const thinkingContent = this.parseThinkingContent(assistantResponse); - if (thinkingContent) { - this.updateThinkingText(thinkingContent); - // Don't display the raw response with think tags in the chat - return; - } - } - // Get the existing assistant message or create a new one let assistantMessageEl = this.noteContextChatMessages.querySelector('.assistant-message:last-child'); @@ -1099,12 +1089,9 @@ export default class LlmChatPanel extends BasicWidget { assistantMessageEl.appendChild(messageContent); } - // Clean the response to remove thinking tags before displaying - const cleanedResponse = this.removeThinkingTags(assistantResponse); - - // Update the content + // Update the content with the current response (no thinking content removal) const messageContent = assistantMessageEl.querySelector('.message-content') as HTMLElement; - messageContent.innerHTML = formatMarkdown(cleanedResponse); + messageContent.innerHTML = formatMarkdown(assistantResponse); // Apply syntax highlighting if this is the final update if (isDone) { @@ -1120,65 +1107,24 @@ export default class LlmChatPanel extends BasicWidget { this.messages.lastIndexOf(assistantMessages[assistantMessages.length - 1]) : -1; if (lastAssistantMsgIndex >= 0) { - // Update existing message with cleaned content - this.messages[lastAssistantMsgIndex].content = cleanedResponse; + // Update existing message + this.messages[lastAssistantMsgIndex].content = assistantResponse; } else { - // Add new message with cleaned content + // Add new message this.messages.push({ role: 'assistant', - content: cleanedResponse + content: assistantResponse }); } - // Hide loading indicator - hideLoadingIndicator(this.loadingIndicator); - - // DON'T save here immediately - let the server save the accumulated response first - // to avoid race conditions. We'll reload the data from the server after a short delay. - console.log("Stream completed, waiting for server to save then reloading data..."); - setTimeout(async () => { - try { - console.log("About to reload data from server..."); - const currentMessageCount = this.messages.length; - console.log(`Current client message count before reload: ${currentMessageCount}`); - - // Reload the data from the server which should have the complete conversation - const reloadSuccess = await this.loadSavedData(); - - const newMessageCount = this.messages.length; - console.log(`Reload success: ${reloadSuccess}, message count after reload: ${newMessageCount}`); - - if (reloadSuccess && newMessageCount > currentMessageCount) { - console.log("Successfully reloaded data with more complete conversation"); - } else if (!reloadSuccess) { - console.warn("Reload failed, keeping current client state"); - } else { - console.warn("Reload succeeded but message count didn't increase"); - } - } catch (error) { - console.error("Failed to reload data after stream completion:", error); - // Fallback: save our current state if reload fails - this.saveCurrentData().catch(err => { - console.error("Failed to save assistant response to note:", err); - }); - } - }, 1500); // Wait 1.5 seconds for server to complete its save + // Save the data + this.saveCurrentData(); } // Scroll to bottom this.chatContainer.scrollTop = this.chatContainer.scrollHeight; } - /** - * Remove thinking tags from response content - */ - private removeThinkingTags(content: string): string { - if (!content) return content; - - // Remove ... blocks from the content - return content.replace(/[\s\S]*?<\/think>/gi, '').trim(); - } - /** * Handle general errors in the send message flow */ diff --git a/apps/server/src/routes/api/llm.ts b/apps/server/src/routes/api/llm.ts index 05e8ee879..bd56830bb 100644 --- a/apps/server/src/routes/api/llm.ts +++ b/apps/server/src/routes/api/llm.ts @@ -858,92 +858,52 @@ async function streamMessage(req: Request, res: Response) { } } - // Create request parameters for the pipeline - const requestParams = { - chatNoteId: chatNoteId, - content: enhancedContent, - useAdvancedContext: useAdvancedContext === true, - showThinking: showThinking === true, - stream: true // Always stream for this endpoint - }; - - // Create a fake request/response pair to pass to the handler - const fakeReq = { - ...req, - method: 'GET', // Set to GET to indicate streaming - query: { - stream: 'true', // Set stream param - don't use format: 'stream' to avoid confusion - useAdvancedContext: String(useAdvancedContext === true), - showThinking: String(showThinking === true) - }, - params: { - chatNoteId: chatNoteId - }, - // Make sure the enhanced content is available to the handler - body: { - content: enhancedContent, - useAdvancedContext: useAdvancedContext === true, - showThinking: showThinking === true - } - } as unknown as Request; - - // Log to verify correct parameters - log.info(`WebSocket stream settings - useAdvancedContext=${useAdvancedContext === true}, in query=${fakeReq.query.useAdvancedContext}, in body=${fakeReq.body.useAdvancedContext}`); - // Extra safety to ensure the parameters are passed correctly - if (useAdvancedContext === true) { - log.info(`Enhanced context IS enabled for this request`); - } else { - log.info(`Enhanced context is NOT enabled for this request`); - } - - // Process the request in the background - Promise.resolve().then(async () => { - try { - await restChatService.handleSendMessage(fakeReq, res); - } catch (error) { - log.error(`Background message processing error: ${error}`); - - // Import the WebSocket service - const wsService = (await import('../../services/ws.js')).default; - - // Define LLMStreamMessage interface - interface LLMStreamMessage { - type: 'llm-stream'; - chatNoteId: string; - content?: string; - thinking?: string; - toolExecution?: any; - done?: boolean; - error?: string; - raw?: unknown; - } - - // Send error to client via WebSocket - wsService.sendMessageToAllClients({ - type: 'llm-stream', - chatNoteId: chatNoteId, - error: `Error processing message: ${error}`, - done: true - } as LLMStreamMessage); - } - }); - - // Import the WebSocket service + // Import the WebSocket service to send immediate feedback const wsService = (await import('../../services/ws.js')).default; - // Let the client know streaming has started via WebSocket (helps client confirm connection is working) + // Let the client know streaming has started wsService.sendMessageToAllClients({ type: 'llm-stream', chatNoteId: chatNoteId, - thinking: 'Initializing streaming LLM response...' + thinking: showThinking ? 'Initializing streaming LLM response...' : undefined }); - // Let the client know streaming has started via HTTP response - return { - success: true, - message: 'Streaming started', - chatNoteId: chatNoteId - }; + // Process the streaming request directly + try { + const result = await restChatService.handleSendMessage({ + ...req, + method: 'GET', // Indicate streaming mode + query: { + ...req.query, + stream: 'true' // Add the required stream parameter + }, + body: { + content: enhancedContent, + useAdvancedContext: useAdvancedContext === true, + showThinking: showThinking === true + }, + params: { chatNoteId } + } as unknown as Request, res); + + // Since we're streaming, the result will be null + return { + success: true, + message: 'Streaming started', + chatNoteId: chatNoteId + }; + } catch (error) { + log.error(`Error during streaming: ${error}`); + + // Send error to client via WebSocket + wsService.sendMessageToAllClients({ + type: 'llm-stream', + chatNoteId: chatNoteId, + error: `Error processing message: ${error}`, + done: true + }); + + throw error; + } } catch (error: any) { log.error(`Error starting message stream: ${error.message}`); throw error; diff --git a/apps/server/src/services/llm/chat/rest_chat_service.ts b/apps/server/src/services/llm/chat/rest_chat_service.ts index 53c682862..2f67422f3 100644 --- a/apps/server/src/services/llm/chat/rest_chat_service.ts +++ b/apps/server/src/services/llm/chat/rest_chat_service.ts @@ -231,21 +231,16 @@ class RestChatService { if (data) { message.content = data; - // Handle accumulation carefully - if this appears to be a complete response - // (done=true and data is much longer than current accumulated), replace rather than append - if (done && data.length > accumulatedContentRef.value.length && data.includes(accumulatedContentRef.value)) { - // This looks like a complete final response that includes what we've accumulated - accumulatedContentRef.value = data; - } else { - // Normal incremental accumulation - accumulatedContentRef.value += data; - } + // Simple accumulation - just append the new data + accumulatedContentRef.value += data; } + // Only include thinking if explicitly present in rawChunk if (rawChunk && 'thinking' in rawChunk && rawChunk.thinking) { message.thinking = rawChunk.thinking as string; } + // Only include tool execution if explicitly present in rawChunk if (rawChunk && 'toolExecution' in rawChunk && rawChunk.toolExecution) { const toolExec = rawChunk.toolExecution; message.toolExecution = { @@ -262,7 +257,7 @@ class RestChatService { // Send WebSocket message wsService.sendMessageToAllClients(message); - // Send SSE response + // Send SSE response for compatibility const responseData: any = { content: data, done }; if (rawChunk?.toolExecution) { responseData.toolExecution = rawChunk.toolExecution; From ab3758c9b3294a256842ed35372d9508333842b6 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Mon, 2 Jun 2025 23:54:38 +0000 Subject: [PATCH 14/18] refactor(llm): resolve issue with headers being sent after request was sent --- apps/server/src/routes/api/llm.ts | 49 ++++++++++++++----- .../services/llm/chat/rest_chat_service.ts | 49 +++++++++++++------ 2 files changed, 69 insertions(+), 29 deletions(-) diff --git a/apps/server/src/routes/api/llm.ts b/apps/server/src/routes/api/llm.ts index bd56830bb..9aaf2fe81 100644 --- a/apps/server/src/routes/api/llm.ts +++ b/apps/server/src/routes/api/llm.ts @@ -809,11 +809,19 @@ async function indexNote(req: Request, res: Response) { async function streamMessage(req: Request, res: Response) { log.info("=== Starting streamMessage ==="); try { + // Set up the response headers for streaming first, before any data is sent + res.setHeader('Content-Type', 'text/event-stream'); + res.setHeader('Cache-Control', 'no-cache'); + res.setHeader('Connection', 'keep-alive'); + const chatNoteId = req.params.chatNoteId; const { content, useAdvancedContext, showThinking, mentions } = req.body; if (!content || typeof content !== 'string' || content.trim().length === 0) { - throw new Error('Content cannot be empty'); + // Early return with error + res.write(`data: ${JSON.stringify({ error: 'Content cannot be empty', done: true })}\n\n`); + res.end(); + return; } // Get or create chat directly from storage (simplified approach) @@ -823,6 +831,14 @@ async function streamMessage(req: Request, res: Response) { chat = await chatStorageService.createChat('New Chat'); log.info(`Created new chat with ID: ${chat.id} for stream request`); } + + // Add the user message to the chat immediately + chat.messages.push({ + role: 'user', + content + }); + // Save the chat to ensure the user message is recorded + await chatStorageService.updateChat(chat.id, chat.messages, chat.title); // Process mentions if provided let enhancedContent = content; @@ -831,7 +847,6 @@ async function streamMessage(req: Request, res: Response) { // Import note service to get note content const becca = (await import('../../becca/becca.js')).default; - const mentionContexts: string[] = []; for (const mention of mentions) { @@ -870,7 +885,10 @@ async function streamMessage(req: Request, res: Response) { // Process the streaming request directly try { - const result = await restChatService.handleSendMessage({ + // Call the streaming handler - it will handle the response streaming + // IMPORTANT: We do not await this because we don't want to try sending a response + // after the streaming has completed - the stream handler takes care of ending the response + restChatService.handleSendMessage({ ...req, method: 'GET', // Indicate streaming mode query: { @@ -884,13 +902,9 @@ async function streamMessage(req: Request, res: Response) { }, params: { chatNoteId } } as unknown as Request, res); - - // Since we're streaming, the result will be null - return { - success: true, - message: 'Streaming started', - chatNoteId: chatNoteId - }; + + // Don't return or send any additional response here + // handleSendMessage handles the full streaming response cycle and will end the response } catch (error) { log.error(`Error during streaming: ${error}`); @@ -901,12 +915,21 @@ async function streamMessage(req: Request, res: Response) { error: `Error processing message: ${error}`, done: true }); - - throw error; + + // Only write to the response if it hasn't been ended yet + if (!res.writableEnded) { + res.write(`data: ${JSON.stringify({ error: `Error processing message: ${error}`, done: true })}\n\n`); + res.end(); + } } } catch (error: any) { log.error(`Error starting message stream: ${error.message}`); - throw error; + + // Only write to the response if it hasn't been ended yet + if (!res.writableEnded) { + res.write(`data: ${JSON.stringify({ error: `Error starting message stream: ${error.message}`, done: true })}\n\n`); + res.end(); + } } } diff --git a/apps/server/src/services/llm/chat/rest_chat_service.ts b/apps/server/src/services/llm/chat/rest_chat_service.ts index 2f67422f3..1ad3d7a22 100644 --- a/apps/server/src/services/llm/chat/rest_chat_service.ts +++ b/apps/server/src/services/llm/chat/rest_chat_service.ts @@ -116,13 +116,16 @@ class RestChatService { throw new Error('Failed to create or retrieve chat'); } - // For POST requests, add the user message + // For POST requests, add the user message to the chat immediately + // This ensures user messages are always saved if (req.method === 'POST' && content) { chat.messages.push({ role: 'user', content }); - log.info(`Processing LLM message: "${content.substring(0, 50)}${content.length > 50 ? '...' : ''}"`); + // Save immediately to ensure user message is saved + await chatStorageService.updateChat(chat.id, chat.messages, chat.title); + log.info(`Added and saved user message: "${content.substring(0, 50)}${content.length > 50 ? '...' : ''}"`); } // Initialize tools @@ -162,7 +165,7 @@ class RestChatService { showThinking: showThinking, options: pipelineOptions, streamCallback: req.method === 'GET' ? (data, done, rawChunk) => { - this.handleStreamCallback(data, done, rawChunk, wsService.default, chatNoteId, res, accumulatedContentRef); + this.handleStreamCallback(data, done, rawChunk, wsService.default, chatNoteId, res, accumulatedContentRef, chat); } : undefined }; @@ -178,6 +181,7 @@ class RestChatService { // Save the updated chat back to storage (single source of truth) await chatStorageService.updateChat(chat.id, chat.messages, chat.title); + log.info(`Saved non-streaming assistant response: ${(response.text || '').length} characters`); // Extract sources if available const sources = (response as any).sources || []; @@ -193,16 +197,7 @@ class RestChatService { }; } else { // For streaming, response is already sent via WebSocket/SSE - // Save the accumulated content - prefer accumulated content over response.text - const finalContent = accumulatedContentRef.value || response.text || ''; - if (finalContent) { - chat.messages.push({ - role: 'assistant', - content: finalContent - }); - await chatStorageService.updateChat(chat.id, chat.messages, chat.title); - log.info(`Saved accumulated streaming content: ${finalContent.length} characters`); - } + // The accumulatedContentRef will have been saved in handleStreamCallback when done=true return null; } } catch (error: any) { @@ -214,14 +209,15 @@ class RestChatService { /** * Simplified stream callback handler */ - private handleStreamCallback( + private async handleStreamCallback( data: string | null, done: boolean, rawChunk: any, wsService: any, chatNoteId: string, res: Response, - accumulatedContentRef: { value: string } + accumulatedContentRef: { value: string }, + chat: { id: string; messages: Message[]; title: string } ) { const message: LLMStreamMessage = { type: 'llm-stream', @@ -264,7 +260,28 @@ class RestChatService { } res.write(`data: ${JSON.stringify(responseData)}\n\n`); + + // When streaming is complete, save the accumulated content to the chat note if (done) { + try { + // Only save if we have accumulated content + if (accumulatedContentRef.value) { + // Add assistant response to chat + chat.messages.push({ + role: 'assistant', + content: accumulatedContentRef.value + }); + + // Save the updated chat back to storage + await chatStorageService.updateChat(chat.id, chat.messages, chat.title); + log.info(`Saved streaming assistant response: ${accumulatedContentRef.value.length} characters`); + } + } catch (error) { + // Log error but don't break the response flow + log.error(`Error saving streaming response: ${error}`); + } + + // End the response res.end(); } } @@ -295,7 +312,7 @@ class RestChatService { log.info(`Using existing AI Chat note ${noteId} as session`); } } - } catch (e) { + } catch (_) { // Not JSON content, so not an AI Chat note } } From d2ba270fdff739dd994b942014b10f0250a032ce Mon Sep 17 00:00:00 2001 From: perf3ct Date: Tue, 3 Jun 2025 00:18:45 +0000 Subject: [PATCH 15/18] fix(llm): sending messages no longer throws an error at first --- apps/server/src/routes/api/llm.ts | 106 ++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 29 deletions(-) diff --git a/apps/server/src/routes/api/llm.ts b/apps/server/src/routes/api/llm.ts index 9aaf2fe81..cec29ffec 100644 --- a/apps/server/src/routes/api/llm.ts +++ b/apps/server/src/routes/api/llm.ts @@ -809,20 +809,26 @@ async function indexNote(req: Request, res: Response) { async function streamMessage(req: Request, res: Response) { log.info("=== Starting streamMessage ==="); try { - // Set up the response headers for streaming first, before any data is sent - res.setHeader('Content-Type', 'text/event-stream'); - res.setHeader('Cache-Control', 'no-cache'); - res.setHeader('Connection', 'keep-alive'); - const chatNoteId = req.params.chatNoteId; const { content, useAdvancedContext, showThinking, mentions } = req.body; if (!content || typeof content !== 'string' || content.trim().length === 0) { - // Early return with error - res.write(`data: ${JSON.stringify({ error: 'Content cannot be empty', done: true })}\n\n`); - res.end(); - return; + return res.status(400).json({ + success: false, + error: 'Content cannot be empty' + }); } + + // IMPORTANT: Immediately send a success response to the initial POST request + // The client is waiting for this to confirm streaming has been initiated + res.status(200).json({ + success: true, + message: 'Streaming initiated successfully' + }); + log.info(`Sent immediate success response for streaming setup`); + + // Create a new response object for streaming through WebSocket only + // We won't use HTTP streaming since we've already sent the HTTP response // Get or create chat directly from storage (simplified approach) let chat = await chatStorageService.getChat(chatNoteId); @@ -883,28 +889,70 @@ async function streamMessage(req: Request, res: Response) { thinking: showThinking ? 'Initializing streaming LLM response...' : undefined }); - // Process the streaming request directly + // Instead of trying to reimplement the streaming logic ourselves, + // delegate to restChatService but set up the correct protocol: + // 1. We've already sent a success response to the initial POST + // 2. Now we'll have restChatService process the actual streaming through WebSocket try { - // Call the streaming handler - it will handle the response streaming - // IMPORTANT: We do not await this because we don't want to try sending a response - // after the streaming has completed - the stream handler takes care of ending the response - restChatService.handleSendMessage({ - ...req, - method: 'GET', // Indicate streaming mode - query: { - ...req.query, - stream: 'true' // Add the required stream parameter - }, - body: { - content: enhancedContent, - useAdvancedContext: useAdvancedContext === true, - showThinking: showThinking === true - }, - params: { chatNoteId } - } as unknown as Request, res); + // Import the WebSocket service for sending messages + const wsService = (await import('../../services/ws.js')).default; - // Don't return or send any additional response here - // handleSendMessage handles the full streaming response cycle and will end the response + // Create a simple pass-through response object that won't write to the HTTP response + // but will allow restChatService to send WebSocket messages + const dummyResponse = { + writableEnded: false, + // Implement methods that would normally be used by restChatService + write: (_chunk: string) => { + // Silent no-op - we're only using WebSocket + return true; + }, + end: (_chunk?: string) => { + // Log when streaming is complete via WebSocket + log.info(`[${chatNoteId}] Completed HTTP response handling during WebSocket streaming`); + return dummyResponse; + }, + setHeader: (name: string, _value: string) => { + // Only log for content-type to reduce noise + if (name.toLowerCase() === 'content-type') { + log.info(`[${chatNoteId}] Setting up streaming for WebSocket only`); + } + return dummyResponse; + } + }; + + // Process the streaming now through WebSocket only + try { + log.info(`[${chatNoteId}] Processing LLM streaming through WebSocket after successful initiation at ${new Date().toISOString()}`); + + // Call restChatService with our enhanced request and dummy response + // The important part is setting method to GET to indicate streaming mode + await restChatService.handleSendMessage({ + ...req, + method: 'GET', // Indicate streaming mode + query: { + ...req.query, + stream: 'true' // Add the required stream parameter + }, + body: { + content: enhancedContent, + useAdvancedContext: useAdvancedContext === true, + showThinking: showThinking === true + }, + params: { chatNoteId } + } as unknown as Request, dummyResponse as unknown as Response); + + log.info(`[${chatNoteId}] WebSocket streaming completed at ${new Date().toISOString()}`); + } catch (streamError) { + log.error(`[${chatNoteId}] Error during WebSocket streaming: ${streamError}`); + + // Send error message through WebSocket + wsService.sendMessageToAllClients({ + type: 'llm-stream', + chatNoteId: chatNoteId, + error: `Error during streaming: ${streamError}`, + done: true + }); + } } catch (error) { log.error(`Error during streaming: ${error}`); From d4d55b20a8ce5a1eb62e9383b4ae62b9ff9015e1 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Tue, 3 Jun 2025 03:00:15 +0000 Subject: [PATCH 16/18] fix(llm): get rid of a lot of log.info() statements that were spammy --- apps/server/src/services/llm/context_extractors/index.ts | 1 - .../src/services/llm/pipeline/stages/tool_calling_stage.ts | 7 +------ apps/server/src/services/llm/tools/search_notes_tool.ts | 4 +--- apps/server/src/services/llm/tools/tool_registry.ts | 3 +-- 4 files changed, 3 insertions(+), 12 deletions(-) diff --git a/apps/server/src/services/llm/context_extractors/index.ts b/apps/server/src/services/llm/context_extractors/index.ts index f6cd07b28..9b97f38a3 100644 --- a/apps/server/src/services/llm/context_extractors/index.ts +++ b/apps/server/src/services/llm/context_extractors/index.ts @@ -42,7 +42,6 @@ export class AgentToolsManager { } try { - log.info("Initializing agent tools"); // Initialize the context service first try { diff --git a/apps/server/src/services/llm/pipeline/stages/tool_calling_stage.ts b/apps/server/src/services/llm/pipeline/stages/tool_calling_stage.ts index 1dd6ff550..f988a6394 100644 --- a/apps/server/src/services/llm/pipeline/stages/tool_calling_stage.ts +++ b/apps/server/src/services/llm/pipeline/stages/tool_calling_stage.ts @@ -559,11 +559,9 @@ export class ToolCallingStage extends BasePipelineStage { // Get agent tools manager and initialize it const agentTools = aiServiceManager.getAgentTools(); if (agentTools && typeof agentTools.initialize === 'function') { - log.info('Initializing agent tools to create vectorSearchTool'); try { // Force initialization to ensure it runs even if previously marked as initialized await agentTools.initialize(true); - log.info('Agent tools initialized successfully'); } catch (initError: any) { log.error(`Failed to initialize agent tools: ${initError.message}`); return null; @@ -143,7 +141,7 @@ export class SearchNotesTool implements ToolHandler { temperature: 0.3, maxTokens: 200, // Type assertion to bypass type checking for special internal parameters - ...(({ + ...(({ bypassFormatter: true, bypassContextProcessing: true } as Record)) diff --git a/apps/server/src/services/llm/tools/tool_registry.ts b/apps/server/src/services/llm/tools/tool_registry.ts index 9ad41fdce..6d6dd417f 100644 --- a/apps/server/src/services/llm/tools/tool_registry.ts +++ b/apps/server/src/services/llm/tools/tool_registry.ts @@ -13,7 +13,7 @@ import log from '../../log.js'; export class ToolRegistry { private static instance: ToolRegistry; private tools: Map = new Map(); - private initializationAttempted: boolean = false; + private initializationAttempted = false; private constructor() {} @@ -106,7 +106,6 @@ export class ToolRegistry { } this.tools.set(name, handler); - log.info(`Registered tool: ${name}`); } /** From 336cd1fbdae123efcd99be43a6df433df74f4d18 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Tue, 3 Jun 2025 03:12:09 +0000 Subject: [PATCH 17/18] fix(llm): storing >1 message in a chat note works fix(llm): storing >1 message in a chat note works --- .../src/widgets/llm_chat/llm_chat_panel.ts | 51 ++++++++++--------- apps/server/src/routes/api/llm.ts | 13 +---- 2 files changed, 27 insertions(+), 37 deletions(-) diff --git a/apps/client/src/widgets/llm_chat/llm_chat_panel.ts b/apps/client/src/widgets/llm_chat/llm_chat_panel.ts index ce9b5dd12..4565c0db2 100644 --- a/apps/client/src/widgets/llm_chat/llm_chat_panel.ts +++ b/apps/client/src/widgets/llm_chat/llm_chat_panel.ts @@ -1068,13 +1068,19 @@ export default class LlmChatPanel extends BasicWidget { * Update the UI with streaming content */ private updateStreamingUI(assistantResponse: string, isDone: boolean = false) { - // Get the existing assistant message or create a new one - let assistantMessageEl = this.noteContextChatMessages.querySelector('.assistant-message:last-child'); - - if (!assistantMessageEl) { - // If no assistant message yet, create one + // Track if we have a streaming message in progress + const hasStreamingMessage = !!this.noteContextChatMessages.querySelector('.assistant-message.streaming'); + + // Create a new message element or use the existing streaming one + let assistantMessageEl: HTMLElement; + + if (hasStreamingMessage) { + // Use the existing streaming message + assistantMessageEl = this.noteContextChatMessages.querySelector('.assistant-message.streaming')!; + } else { + // Create a new message element assistantMessageEl = document.createElement('div'); - assistantMessageEl.className = 'assistant-message message mb-3'; + assistantMessageEl.className = 'assistant-message message mb-3 streaming'; this.noteContextChatMessages.appendChild(assistantMessageEl); // Add assistant profile icon @@ -1089,35 +1095,30 @@ export default class LlmChatPanel extends BasicWidget { assistantMessageEl.appendChild(messageContent); } - // Update the content with the current response (no thinking content removal) + // Update the content with the current response const messageContent = assistantMessageEl.querySelector('.message-content') as HTMLElement; messageContent.innerHTML = formatMarkdown(assistantResponse); - // Apply syntax highlighting if this is the final update + // When the response is complete if (isDone) { + // Remove the streaming class to mark this message as complete + assistantMessageEl.classList.remove('streaming'); + + // Apply syntax highlighting formatCodeBlocks($(assistantMessageEl as HTMLElement)); // Hide the thinking display when response is complete this.hideThinkingDisplay(); - // Update message in the data model for storage - // Find the last assistant message to update, or add a new one if none exists - const assistantMessages = this.messages.filter(msg => msg.role === 'assistant'); - const lastAssistantMsgIndex = assistantMessages.length > 0 ? - this.messages.lastIndexOf(assistantMessages[assistantMessages.length - 1]) : -1; + // Always add a new message to the data model + // This ensures we preserve all distinct assistant messages + this.messages.push({ + role: 'assistant', + content: assistantResponse, + timestamp: new Date() + }); - if (lastAssistantMsgIndex >= 0) { - // Update existing message - this.messages[lastAssistantMsgIndex].content = assistantResponse; - } else { - // Add new message - this.messages.push({ - role: 'assistant', - content: assistantResponse - }); - } - - // Save the data + // Save the updated message list this.saveCurrentData(); } diff --git a/apps/server/src/routes/api/llm.ts b/apps/server/src/routes/api/llm.ts index cec29ffec..c21426a66 100644 --- a/apps/server/src/routes/api/llm.ts +++ b/apps/server/src/routes/api/llm.ts @@ -963,21 +963,10 @@ async function streamMessage(req: Request, res: Response) { error: `Error processing message: ${error}`, done: true }); - - // Only write to the response if it hasn't been ended yet - if (!res.writableEnded) { - res.write(`data: ${JSON.stringify({ error: `Error processing message: ${error}`, done: true })}\n\n`); - res.end(); - } } } catch (error: any) { log.error(`Error starting message stream: ${error.message}`); - - // Only write to the response if it hasn't been ended yet - if (!res.writableEnded) { - res.write(`data: ${JSON.stringify({ error: `Error starting message stream: ${error.message}`, done: true })}\n\n`); - res.end(); - } + log.error(`Error starting message stream, can't communicate via WebSocket: ${error.message}`); } } From b76166b0d5c87b9f433f32aa6fadecdb4e17e19c Mon Sep 17 00:00:00 2001 From: perf3ct Date: Tue, 3 Jun 2025 05:13:32 +0000 Subject: [PATCH 18/18] fix(llm): always fetch the embedding model --- apps/server/src/services/llm/providers/providers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/services/llm/providers/providers.ts b/apps/server/src/services/llm/providers/providers.ts index 91295ffe0..f4d69801c 100644 --- a/apps/server/src/services/llm/providers/providers.ts +++ b/apps/server/src/services/llm/providers/providers.ts @@ -300,7 +300,7 @@ export async function initializeDefaultProviders() { const ollamaBaseUrl = await options.getOption('ollamaBaseUrl'); if (ollamaBaseUrl) { // Use specific embedding models if available - const embeddingModel = await options.getOption('ollamaEmbeddingModel') || 'nomic-embed-text'; + const embeddingModel = await options.getOption('ollamaEmbeddingModel'); try { // Create provider with initial dimension to be updated during initialization