From 511d2c5e96f159076ad6569caac5330a8a0b49c6 Mon Sep 17 00:00:00 2001
From: perf3ct <jonfuller2012@gmail.com>
Date: Thu, 29 May 2025 21:04:06 +0000
Subject: [PATCH] feat(llm): add empty result handling and parameter adjustment
 suggestions for tool execution

---
 .../prompts/providers/ollama_tool_prompt.md   |  11 ++
 .../llm/pipeline/stages/tool_calling_stage.ts | 105 +++++++++++++++++-
 2 files changed, 111 insertions(+), 5 deletions(-)

diff --git a/apps/server/src/assets/llm/prompts/providers/ollama_tool_prompt.md b/apps/server/src/assets/llm/prompts/providers/ollama_tool_prompt.md
index c89760444..499369ea8 100644
--- a/apps/server/src/assets/llm/prompts/providers/ollama_tool_prompt.md
+++ b/apps/server/src/assets/llm/prompts/providers/ollama_tool_prompt.md
@@ -32,4 +32,15 @@ When responding to queries:
 5. For general questions about the user's notes, provide a summary of all relevant notes found, including brief summaries of individual notes
 6. For specific questions, provide detailed information from the user's notes that directly addresses the question
 7. Always prioritize information from the user's notes over your own knowledge, as the user's notes are likely more up-to-date and personally relevant
+
+When using tools, follow these best practices:
+1. If a tool returns an error or no results, DO NOT give up immediately
+2. Instead, try different parameters that might yield better results:
+   - For search tools: Try broader search terms, fewer filters, or synonyms
+   - For note navigation: Try parent or sibling notes if a specific note isn't found
+   - For content analysis: Try rephrasing or generalizing the query
+3. When searching for information, start with specific search terms but be prepared to broaden your search if no results are found
+4. If multiple attempts with different parameters still yield no results, clearly explain to the user that the information they're looking for might not be in their notes
+5. When suggesting alternatives, be explicit about what parameters you've tried and what you're changing
+6. Remember that empty results from tools don't mean the user's request can't be fulfilled - it often means the parameters need adjustment
 ```
\ No newline at end of file
diff --git a/apps/server/src/services/llm/pipeline/stages/tool_calling_stage.ts b/apps/server/src/services/llm/pipeline/stages/tool_calling_stage.ts
index a5a913f24..231bcb3eb 100644
--- a/apps/server/src/services/llm/pipeline/stages/tool_calling_stage.ts
+++ b/apps/server/src/services/llm/pipeline/stages/tool_calling_stage.ts
@@ -364,6 +364,7 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
 
         // Add each tool result to the messages array
         const toolResultMessages: Message[] = [];
+        let hasEmptyResults = false;
 
         for (const result of toolResults) {
             const { toolCallId, name, result: toolResult } = result;
@@ -373,10 +374,23 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
                 ? toolResult
                 : JSON.stringify(toolResult, null, 2);
 
+            // Check if result is empty or unhelpful
+            const isEmptyResult = this.isEmptyToolResult(toolResult, name);
+            if (isEmptyResult && !resultContent.startsWith('Error:')) {
+                hasEmptyResults = true;
+                log.info(`Empty result detected for tool ${name}. Will add suggestion to try different parameters.`);
+            }
+            
+            // Add enhancement for empty results
+            let enhancedContent = resultContent;
+            if (isEmptyResult && !resultContent.startsWith('Error:')) {
+                enhancedContent = `${resultContent}\n\nNOTE: This tool returned no useful results with the provided parameters. Consider trying again with different parameters such as broader search terms, different filters, or alternative approaches.`;
+            }
+
             // Add a new message for the tool result
             const toolMessage: Message = {
                 role: 'tool',
-                content: resultContent,
+                content: enhancedContent,
                 name: name,
                 tool_call_id: toolCallId
             };
@@ -385,7 +399,7 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
             log.info(`-------- Tool Result for ${name} (ID: ${toolCallId}) --------`);
             log.info(`Result type: ${typeof toolResult}`);
             log.info(`Result preview: ${resultContent.substring(0, 150)}${resultContent.length > 150 ? '...' : ''}`);
-            log.info(`Tool result status: ${resultContent.startsWith('Error:') ? 'ERROR' : 'SUCCESS'}`);
+            log.info(`Tool result status: ${resultContent.startsWith('Error:') ? 'ERROR' : isEmptyResult ? 'EMPTY' : 'SUCCESS'}`);
 
             updatedMessages.push(toolMessage);
             toolResultMessages.push(toolMessage);
@@ -398,7 +412,17 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
         const needsFollowUp = hasToolResults;
 
         log.info(`Follow-up needed: ${needsFollowUp}`);
-        log.info(`Reasoning: ${hasToolResults ? 'Has tool results to process' : 'No tool results'} ${hasErrors ? ', contains errors' : ''}`);
+        log.info(`Reasoning: ${hasToolResults ? 'Has tool results to process' : 'No tool results'} ${hasErrors ? ', contains errors' : ''} ${hasEmptyResults ? ', contains empty results' : ''}`);
+        
+        // Add a system message with hints for empty results
+        if (hasEmptyResults && needsFollowUp) {
+            log.info('Adding system hint for empty tool results to encourage parameter adjustment');
+            updatedMessages.push({
+                role: 'system',
+                content: `The previous tool execution(s) completed but returned no useful results. Consider trying different parameters for better results. For search tools, try broader search terms, fewer filters, or synonyms. For navigation tools, try exploring parent or sibling notes.`
+            });
+        }
+        
         log.info(`Total messages to return to pipeline: ${updatedMessages.length}`);
         log.info(`Last 3 messages in conversation:`);
         const lastMessages = updatedMessages.slice(-3);
@@ -421,7 +445,7 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
      * @param toolName The name of the tool requiring this dependency
      * @returns The requested dependency or null if it couldn't be created
      */
-    private async getOrCreateDependency(dependencyType: string, toolName: string): Promise<any> {
+    private async getOrCreateDependency(dependencyType: string, toolName: string): Promise<unknown | null> {
         const aiServiceManager = (await import('../../ai_service_manager.js')).default;
 
         try {
@@ -485,7 +509,7 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
      * @param tool The tool to validate
      * @param toolName The name of the tool
      */
-    private async validateToolBeforeExecution(tool: any, toolName: string): Promise<boolean> {
+    private async validateToolBeforeExecution(tool: { execute: (args: Record<string, unknown>) => Promise<unknown> }, toolName: string): Promise<boolean> {
         try {
             if (!tool) {
                 log.error(`Tool '${toolName}' not found or failed validation`);
@@ -550,6 +574,77 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
         }
     }
 
+    /**
+     * Determines if a tool result is effectively empty or unhelpful
+     * @param result The result from the tool execution
+     * @param toolName The name of the tool that was executed
+     * @returns true if the result is considered empty or unhelpful
+     */
+    private isEmptyToolResult(result: unknown, toolName: string): boolean {
+        // Handle string results
+        if (typeof result === 'string') {
+            const trimmed = result.trim();
+            if (trimmed === '' || trimmed === '[]' || trimmed === '{}') {
+                return true;
+            }
+            
+            // Tool-specific empty results (for string responses)
+            if (toolName === 'search_notes' && 
+                (trimmed === 'No matching notes found.' || 
+                 trimmed.includes('No results found') || 
+                 trimmed.includes('No matches found') ||
+                 trimmed.includes('No notes found'))) {
+                return true;
+            }
+            
+            if (toolName === 'vector_search' && 
+                (trimmed.includes('No results found') ||
+                 trimmed.includes('No matching documents'))) {
+                return true;
+            }
+            
+            if (toolName === 'keyword_search' && 
+                (trimmed.includes('No matches found') ||
+                 trimmed.includes('No results for'))) {
+                return true;
+            }
+        } 
+        // Handle object/array results
+        else if (result !== null && typeof result === 'object') {
+            // Check if it's an empty array
+            if (Array.isArray(result) && result.length === 0) {
+                return true;
+            }
+            
+            // Check if it's an object with no meaningful properties
+            // or with properties indicating empty results
+            if (!Array.isArray(result)) {
+                if (Object.keys(result).length === 0) {
+                    return true;
+                }
+                
+                // Tool-specific object empty checks
+                const resultObj = result as Record<string, unknown>;
+                
+                if (toolName === 'search_notes' && 
+                    'results' in resultObj && 
+                    Array.isArray(resultObj.results) && 
+                    resultObj.results.length === 0) {
+                    return true;
+                }
+                
+                if (toolName === 'vector_search' && 
+                    'matches' in resultObj && 
+                    Array.isArray(resultObj.matches) && 
+                    resultObj.matches.length === 0) {
+                    return true;
+                }
+            }
+        }
+        
+        return false;
+    }
+
     /**
      * Preload the vector search tool to ensure it's available before tool execution
      */