From b05b88dd765d1721e6e86bd5820b30b58858b96f Mon Sep 17 00:00:00 2001 From: perf3ct Date: Wed, 9 Apr 2025 19:53:45 +0000 Subject: [PATCH] yes, this finally does set streaming to true --- src/public/app/widgets/llm_chat_panel.ts | 14 +++++-- src/services/llm/ai_interface.ts | 12 ++++++ src/services/llm/ai_service_manager.ts | 9 +++++ src/services/llm/pipeline/chat_pipeline.ts | 34 +++++++++++++--- .../pipeline/stages/llm_completion_stage.ts | 40 +++++++++++++++++-- .../pipeline/stages/model_selection_stage.ts | 20 ++++++++++ src/services/llm/providers/ollama_service.ts | 18 +++++++-- src/services/llm/providers/providers.ts | 2 +- src/services/llm/rest_chat_service.ts | 4 +- 9 files changed, 134 insertions(+), 19 deletions(-) diff --git a/src/public/app/widgets/llm_chat_panel.ts b/src/public/app/widgets/llm_chat_panel.ts index 25a2bbb4d..c382f5595 100644 --- a/src/public/app/widgets/llm_chat_panel.ts +++ b/src/public/app/widgets/llm_chat_panel.ts @@ -410,8 +410,14 @@ export default class LlmChatPanel extends BasicWidget { */ private async handleDirectResponse(messageParams: any): Promise { try { - // Send the message via POST request - const postResponse = await server.post(`llm/sessions/${this.sessionId}/messages`, messageParams); + // Add format parameter to maintain consistency with the streaming GET request + const postParams = { + ...messageParams, + format: 'stream' // Match the format parameter used in the GET streaming request + }; + + // Send the message via POST request with the updated params + const postResponse = await server.post(`llm/sessions/${this.sessionId}/messages`, postParams); // If the POST request returned content directly, display it if (postResponse && postResponse.content) { @@ -460,8 +466,8 @@ export default class LlmChatPanel extends BasicWidget { const useAdvancedContext = messageParams.useAdvancedContext; const showThinking = messageParams.showThinking; - // Set up streaming via EventSource - const streamUrl = `./api/llm/sessions/${this.sessionId}/messages?format=stream&useAdvancedContext=${useAdvancedContext}&showThinking=${showThinking}`; + // Set up streaming via EventSource - explicitly add stream=true parameter to ensure consistency + const streamUrl = `./api/llm/sessions/${this.sessionId}/messages?format=stream&stream=true&useAdvancedContext=${useAdvancedContext}&showThinking=${showThinking}`; return new Promise((resolve, reject) => { const source = new EventSource(streamUrl); diff --git a/src/services/llm/ai_interface.ts b/src/services/llm/ai_interface.ts index 1bef407e4..69979b3bc 100644 --- a/src/services/llm/ai_interface.ts +++ b/src/services/llm/ai_interface.ts @@ -20,6 +20,18 @@ export interface StreamChunk { }; } +/** + * Options for chat completion requests + * + * Key properties: + * - stream: If true, the response will be streamed + * - model: Model name to use + * - provider: Provider to use (openai, anthropic, ollama, etc.) + * - enableTools: If true, enables tool support + * + * The stream option is particularly important and should be consistently handled + * throughout the pipeline. It should be explicitly set to true or false. + */ export interface ChatCompletionOptions { model?: string; temperature?: number; diff --git a/src/services/llm/ai_service_manager.ts b/src/services/llm/ai_service_manager.ts index 5da202253..12639baa9 100644 --- a/src/services/llm/ai_service_manager.ts +++ b/src/services/llm/ai_service_manager.ts @@ -197,6 +197,13 @@ export class AIServiceManager implements IAIServiceManager { async generateChatCompletion(messages: Message[], options: ChatCompletionOptions = {}): Promise { this.ensureInitialized(); + log.info(`[AIServiceManager] generateChatCompletion called with options: ${JSON.stringify({ + model: options.model, + stream: options.stream, + enableTools: options.enableTools + })}`); + log.info(`[AIServiceManager] Stream option type: ${typeof options.stream}`); + if (!messages || messages.length === 0) { throw new Error('No messages provided for chat completion'); } @@ -219,6 +226,7 @@ export class AIServiceManager implements IAIServiceManager { if (availableProviders.includes(providerName as ServiceProviders)) { try { const modifiedOptions = { ...options, model: modelName }; + log.info(`[AIServiceManager] Using provider ${providerName} from model prefix with modifiedOptions.stream: ${modifiedOptions.stream}`); return await this.services[providerName as ServiceProviders].generateChatCompletion(messages, modifiedOptions); } catch (error) { log.error(`Error with specified provider ${providerName}: ${error}`); @@ -232,6 +240,7 @@ export class AIServiceManager implements IAIServiceManager { for (const provider of sortedProviders) { try { + log.info(`[AIServiceManager] Trying provider ${provider} with options.stream: ${options.stream}`); return await this.services[provider].generateChatCompletion(messages, options); } catch (error) { log.error(`Error with provider ${provider}: ${error}`); diff --git a/src/services/llm/pipeline/chat_pipeline.ts b/src/services/llm/pipeline/chat_pipeline.ts index 7c805440c..21370b55d 100644 --- a/src/services/llm/pipeline/chat_pipeline.ts +++ b/src/services/llm/pipeline/chat_pipeline.ts @@ -227,14 +227,36 @@ export class ChatPipeline { log.info(`Prepared ${preparedMessages.messages.length} messages for LLM, tools enabled: ${useTools}`); // Setup streaming handler if streaming is enabled and callback provided - const enableStreaming = this.config.enableStreaming && - modelSelection.options.stream !== false && - typeof streamCallback === 'function'; - - if (enableStreaming) { - // Make sure stream is enabled in options + // Check if streaming should be enabled based on several conditions + const streamEnabledInConfig = this.config.enableStreaming; + const streamFormatRequested = input.format === 'stream'; + const streamRequestedInOptions = modelSelection.options.stream === true; + const streamCallbackAvailable = typeof streamCallback === 'function'; + + log.info(`[ChatPipeline] Request type info - Format: ${input.format || 'not specified'}, Options from pipelineInput: ${JSON.stringify({stream: input.options?.stream})}`); + log.info(`[ChatPipeline] Stream settings - config.enableStreaming: ${streamEnabledInConfig}, format parameter: ${input.format}, modelSelection.options.stream: ${modelSelection.options.stream}, streamCallback available: ${streamCallbackAvailable}`); + + // IMPORTANT: Different behavior for GET vs POST requests: + // - For GET requests with streamCallback available: Always enable streaming + // - For POST requests: Use streaming options but don't actually stream (since we can't stream back to client) + if (streamCallbackAvailable) { + // If a stream callback is available (GET requests), we can stream the response modelSelection.options.stream = true; + log.info(`[ChatPipeline] Stream callback available, setting stream=true for real-time streaming`); + } else { + // For POST requests, preserve the stream flag as-is from input options + // This ensures LLM request format is consistent across both GET and POST + if (streamRequestedInOptions) { + log.info(`[ChatPipeline] No stream callback but stream requested in options, preserving stream=true`); + } else { + log.info(`[ChatPipeline] No stream callback and no stream in options, setting stream=false`); + modelSelection.options.stream = false; + } } + + log.info(`[ChatPipeline] Final modelSelection.options.stream = ${modelSelection.options.stream}`); + log.info(`[ChatPipeline] Will actual streaming occur? ${streamCallbackAvailable && modelSelection.options.stream}`); + // STAGE 5 & 6: Handle LLM completion and tool execution loop log.info(`========== STAGE 5: LLM COMPLETION ==========`); diff --git a/src/services/llm/pipeline/stages/llm_completion_stage.ts b/src/services/llm/pipeline/stages/llm_completion_stage.ts index 5efe108b0..bff0f8afa 100644 --- a/src/services/llm/pipeline/stages/llm_completion_stage.ts +++ b/src/services/llm/pipeline/stages/llm_completion_stage.ts @@ -1,6 +1,6 @@ import { BasePipelineStage } from '../pipeline_stage.js'; import type { LLMCompletionInput } from '../interfaces.js'; -import type { ChatResponse } from '../../ai_interface.js'; +import type { ChatCompletionOptions, ChatResponse } from '../../ai_interface.js'; import aiServiceManager from '../../ai_service_manager.js'; import toolRegistry from '../../tools/tool_registry.js'; import log from '../../../log.js'; @@ -19,8 +19,35 @@ export class LLMCompletionStage extends BasePipelineStage { const { messages, options, provider } = input; - // Create a copy of options to avoid modifying the original - const updatedOptions = { ...options }; + // Log input options, particularly focusing on the stream option + log.info(`[LLMCompletionStage] Input options: ${JSON.stringify({ + model: options.model, + provider, + stream: options.stream, + enableTools: options.enableTools + })}`); + log.info(`[LLMCompletionStage] Stream option in input: ${options.stream}, type: ${typeof options.stream}`); + + // Create a deep copy of options to avoid modifying the original + const updatedOptions: ChatCompletionOptions = JSON.parse(JSON.stringify(options)); + + // IMPORTANT: Ensure stream property is explicitly set to a boolean value + // This is critical to ensure consistent behavior across all providers + updatedOptions.stream = options.stream === true; + + log.info(`[LLMCompletionStage] Explicitly set stream option to boolean: ${updatedOptions.stream}`); + + // If this is a direct (non-stream) call to Ollama but has the stream flag, + // ensure we set additional metadata to maintain proper state + if (updatedOptions.stream && !provider && updatedOptions.providerMetadata?.provider === 'ollama') { + log.info(`[LLMCompletionStage] This is an Ollama request with stream=true, ensuring provider config is consistent`); + } + + log.info(`[LLMCompletionStage] Copied options: ${JSON.stringify({ + model: updatedOptions.model, + stream: updatedOptions.stream, + enableTools: updatedOptions.enableTools + })}`); // Check if tools should be enabled if (updatedOptions.enableTools !== false) { @@ -48,15 +75,22 @@ export class LLMCompletionStage extends BasePipelineStage { const { options: inputOptions, query, contentLength } = input; + // Log input options + log.info(`[ModelSelectionStage] Input options: ${JSON.stringify({ + model: inputOptions?.model, + stream: inputOptions?.stream, + enableTools: inputOptions?.enableTools + })}`); + log.info(`[ModelSelectionStage] Stream option in input: ${inputOptions?.stream}, type: ${typeof inputOptions?.stream}`); + // Start with provided options or create a new object const updatedOptions: ChatCompletionOptions = { ...(inputOptions || {}) }; + + // Preserve the stream option exactly as it was provided, including undefined state + // This is critical for ensuring the stream option propagates correctly down the pipeline + log.info(`[ModelSelectionStage] After copy, stream: ${updatedOptions.stream}, type: ${typeof updatedOptions.stream}`); // If model already specified, don't override it if (updatedOptions.model) { @@ -36,6 +48,7 @@ export class ModelSelectionStage extends BasePipelineStage { res.write(`data: ${JSON.stringify({ content: data, done })}\n\n`);