feat(llm): add LLM guidance system for failed tool executions and validations

apps/server/src/services/llm/pipeline/stages/tool_calling_stage.ts

@@ -23,6 +23,7 @@ interface ToolValidationResult {
     valid: boolean;
     tool: ToolInterface | null;
     error: string | null;
+    guidance?: string; // Guidance to help the LLM select better tools/parameters
 }
 
 /**
@@ -141,11 +142,14 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
 
                 if (!tool) {
                     log.error(`Tool not found in registry: ${toolCall.function.name}`);
+                    // Generate guidance for the LLM when a tool is not found
+                    const guidance = this.generateToolGuidance(toolCall.function.name, `Tool not found: ${toolCall.function.name}`);
                     return {
                         toolCall,
                         valid: false,
                         tool: null,
-                        error: `Tool not found: ${toolCall.function.name}`
+                        error: `Tool not found: ${toolCall.function.name}`,
+                        guidance // Add guidance for the LLM
                     };
                 }
 
@@ -187,9 +191,15 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
                     : JSON.stringify(toolCall.function.arguments);
                 log.info(`Tool parameters: ${argsStr}`);
 
-                // If validation failed, throw the error
+                // If validation failed, generate guidance and throw the error
                 if (!valid || !tool) {
-                    throw new Error(error || `Unknown validation error for tool '${toolCall.function.name}'`);
+                    // If we already have guidance from validation, use it, otherwise generate it
+                    const toolGuidance = validation.guidance ||
+                        this.generateToolGuidance(toolCall.function.name,
+                            error || `Unknown validation error for tool '${toolCall.function.name}'`);
+
+                    // Include the guidance in the error message
+                    throw new Error(`${error || `Unknown validation error for tool '${toolCall.function.name}'`}\n${toolGuidance}`);
                 }
 
                 log.info(`Tool validated successfully: ${toolCall.function.name}`);
@@ -315,6 +325,12 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
                     const errorMessage = execError instanceof Error ? execError.message : String(execError);
                     log.error(`================ TOOL EXECUTION FAILED in ${executionTime}ms: ${errorMessage} ================`);
 
+                    // Generate guidance for the failed tool execution
+                    const toolGuidance = this.generateToolGuidance(toolCall.function.name, errorMessage);
+
+                    // Add the guidance to the error message for the LLM
+                    const enhancedErrorMessage = `${errorMessage}\n${toolGuidance}`;
+
                     // Record this failed tool execution if there's a sessionId available
                     if (input.options?.sessionId) {
                         try {
@@ -324,7 +340,7 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
                                 toolCall.id || `tool-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`,
                                 args,
                                 "", // No result for failed execution
-                                errorMessage
+                                enhancedErrorMessage // Use enhanced error message with guidance
                             );
                         } catch (storageError) {
                             log.error(`Failed to record tool execution error in chat storage: ${storageError}`);
@@ -339,7 +355,7 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
                                 name: toolCall.function.name,
                                 arguments: {} as Record<string, unknown>
                             },
-                            error: errorMessage,
+                            error: enhancedErrorMessage, // Include guidance in the error message
                             type: 'error' as const
                         };
 
@@ -354,6 +370,10 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
                         }
                     }
 
+                    // Modify the error to include our guidance
+                    if (execError instanceof Error) {
+                        execError.message = enhancedErrorMessage;
+                    }
                     throw execError;
                 }
 
@@ -544,8 +564,9 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
                         // Force initialization to ensure it runs even if previously marked as initialized
                         await agentTools.initialize(true);
                         log.info('Agent tools initialized successfully');
-                    } catch (initError: any) {
+                    } catch (initError: unknown) {
-                        log.error(`Failed to initialize agent tools: ${initError.message}`);
+                        const errorMessage = initError instanceof Error ? initError.message : String(initError);
+                        log.error(`Failed to initialize agent tools: ${errorMessage}`);
                         return null;
                     }
                 } else {
@@ -629,6 +650,7 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
                         }
                     }
 
+                    // Verify the vectorSearchTool has the required methods
                     if (!vectorSearchTool.searchNotes || typeof vectorSearchTool.searchNotes !== 'function') {
                         log.error(`Tool '${toolName}' dependency vectorSearchTool is missing searchNotes method`);
                         return false;
@@ -641,7 +663,6 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
             }
 
             // Add additional tool-specific validations here
-
             return true;
         } catch (error: unknown) {
             const errorMessage = error instanceof Error ? error.message : String(error);
@@ -650,6 +671,63 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
         }
     }
 
+    /**
+     * Generate guidance for the LLM when a tool fails or is not found
+     * @param toolName The name of the tool that failed
+     * @param errorMessage The error message from the failed tool
+     * @returns A guidance message for the LLM with suggestions of what to try next
+     */
+    private generateToolGuidance(toolName: string, errorMessage: string): string {
+        // Get all available tool names for recommendations
+        const availableTools = toolRegistry.getAllTools();
+        const availableToolNames = availableTools
+            .map(t => {
+                if (t && typeof t === 'object' && 'definition' in t &&
+                    t.definition && typeof t.definition === 'object' &&
+                    'function' in t.definition && t.definition.function &&
+                    typeof t.definition.function === 'object' &&
+                    'name' in t.definition.function &&
+                    typeof t.definition.function.name === 'string') {
+                    return t.definition.function.name;
+                }
+                return '';
+            })
+            .filter(name => name !== '');
+
+        // Create specific guidance based on the error and tool
+        let guidance = `TOOL GUIDANCE: The tool '${toolName}' failed with error: ${errorMessage}.\n`;
+
+        // Add suggestions based on the specific tool and error
+        if (toolName === 'attribute_search' && errorMessage.includes('Invalid attribute type')) {
+            guidance += "CORRECT USAGE: The 'attribute_search' tool requires a valid 'attributeType' parameter that must be either 'label' or 'relation'.\n";
+            guidance += "EXAMPLE: { \"attributeType\": \"label\", \"attributeValue\": \"important\" }\n";
+        }
+        else if (errorMessage.includes('Tool not found')) {
+            // Provide guidance on available search tools if a tool wasn't found
+            const searchTools = availableToolNames.filter(name => name.includes('search'));
+            guidance += `AVAILABLE SEARCH TOOLS: ${searchTools.join(', ')}\n`;
+            guidance += "TRY VECTOR SEARCH: For conceptual matches, use 'vector_search' with a query parameter.\n";
+            guidance += "EXAMPLE: { \"query\": \"your search terms here\" }\n";
+        }
+        else if (errorMessage.includes('missing required parameter')) {
+            // Provide parameter guidance based on the tool name
+            if (toolName === 'vector_search') {
+                guidance += "REQUIRED PARAMETERS: The 'vector_search' tool requires a 'query' parameter.\n";
+                guidance += "EXAMPLE: { \"query\": \"your search terms here\" }\n";
+            } else if (toolName === 'keyword_search') {
+                guidance += "REQUIRED PARAMETERS: The 'keyword_search' tool requires a 'query' parameter.\n";
+                guidance += "EXAMPLE: { \"query\": \"your search terms here\" }\n";
+            }
+        }
+
+        // Add a general suggestion to try vector_search as a fallback
+        if (!toolName.includes('vector_search')) {
+            guidance += "RECOMMENDATION: If specific searches fail, try the 'vector_search' tool which performs semantic searches.\n";
+        }
+
+        return guidance;
+    }
+
     /**
      * Determines if a tool result is effectively empty or unhelpful
      * @param result The result from the tool execution