Tools/Notes
1
0
Fork 0
mirror of https://github.com/TriliumNext/Notes.git synced 2025-07-29 19:12:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(llm): add LLM guidance system for failed tool executions and validations

Browse Source
This commit is contained in:
perf3ct 2025-05-30 00:15:40 +00:00
parent f04e56137b
commit 910c5039f4
No known key found for this signature in database
GPG Key ID: 569C4EEC436F5232
1 changed files with 123 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
apps/server/src/services/llm/pipeline/stages/tool_calling_stage.ts
View File

@ -23,6 +23,7 @@ interface ToolValidationResult {
valid: boolean;
tool: ToolInterface | null;
error: string | null;
guidance?: string; // Guidance to help the LLM select better tools/parameters
}
/**
@ -141,11 +142,14 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
if (!tool) {
log.error(`Tool not found in registry: ${toolCall.function.name}`);
// Generate guidance for the LLM when a tool is not found
const guidance = this.generateToolGuidance(toolCall.function.name, `Tool not found: ${toolCall.function.name}`);
return {
toolCall,
valid: false,
tool: null,
error: `Tool not found: ${toolCall.function.name}`
error: `Tool not found: ${toolCall.function.name}`,
guidance // Add guidance for the LLM
};
}
@ -187,9 +191,15 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
: JSON.stringify(toolCall.function.arguments);
log.info(`Tool parameters: ${argsStr}`);
// If validation failed, throw the error
// If validation failed, generate guidance and throw the error
if (!valid || !tool) {
throw new Error(error || `Unknown validation error for tool '${toolCall.function.name}'`);
// If we already have guidance from validation, use it, otherwise generate it
const toolGuidance = validation.guidance ||
this.generateToolGuidance(toolCall.function.name,
error || `Unknown validation error for tool '${toolCall.function.name}'`);
// Include the guidance in the error message
throw new Error(`${error || `Unknown validation error for tool '${toolCall.function.name}'`}\n${toolGuidance}`);
}
log.info(`Tool validated successfully: ${toolCall.function.name}`);
@ -315,6 +325,12 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
const errorMessage = execError instanceof Error ? execError.message : String(execError);
log.error(`================ TOOL EXECUTION FAILED in ${executionTime}ms: ${errorMessage} ================`);
// Generate guidance for the failed tool execution
const toolGuidance = this.generateToolGuidance(toolCall.function.name, errorMessage);
// Add the guidance to the error message for the LLM
const enhancedErrorMessage = `${errorMessage}\n${toolGuidance}`;
// Record this failed tool execution if there's a sessionId available
if (input.options?.sessionId) {
try {
@ -324,7 +340,7 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
toolCall.id || `tool-${Date.now()}-${Math.random().toString(36).substring(2, 9)}`,
args,
"", // No result for failed execution
errorMessage
enhancedErrorMessage // Use enhanced error message with guidance
);
} catch (storageError) {
log.error(`Failed to record tool execution error in chat storage: ${storageError}`);
@ -339,7 +355,7 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
name: toolCall.function.name,
arguments: {} as Record<string, unknown>
},
error: errorMessage,
error: enhancedErrorMessage, // Include guidance in the error message
type: 'error' as const
};
@ -354,6 +370,10 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
}
}
// Modify the error to include our guidance
if (execError instanceof Error) {
execError.message = enhancedErrorMessage;
}
throw execError;
}
@ -544,8 +564,9 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
// Force initialization to ensure it runs even if previously marked as initialized
await agentTools.initialize(true);
log.info('Agent tools initialized successfully');
} catch (initError: any) {
log.error(`Failed to initialize agent tools: ${initError.message}`);
} catch (initError: unknown) {
const errorMessage = initError instanceof Error ? initError.message : String(initError);
log.error(`Failed to initialize agent tools: ${errorMessage}`);
return null;
}
} else {
@ -629,6 +650,7 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
}
}
// Verify the vectorSearchTool has the required methods
if (!vectorSearchTool.searchNotes || typeof vectorSearchTool.searchNotes !== 'function') {
log.error(`Tool '${toolName}' dependency vectorSearchTool is missing searchNotes method`);
return false;
@ -641,7 +663,6 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
}
// Add additional tool-specific validations here
return true;
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : String(error);
@ -650,6 +671,63 @@ export class ToolCallingStage extends BasePipelineStage<ToolExecutionInput, { re
}
}
/**
* Generate guidance for the LLM when a tool fails or is not found
* @param toolName The name of the tool that failed
* @param errorMessage The error message from the failed tool
* @returns A guidance message for the LLM with suggestions of what to try next
*/
private generateToolGuidance(toolName: string, errorMessage: string): string {
// Get all available tool names for recommendations
const availableTools = toolRegistry.getAllTools();
const availableToolNames = availableTools
.map(t => {
if (t && typeof t === 'object' && 'definition' in t &&
t.definition && typeof t.definition === 'object' &&
'function' in t.definition && t.definition.function &&
typeof t.definition.function === 'object' &&
'name' in t.definition.function &&
typeof t.definition.function.name === 'string') {
return t.definition.function.name;
}
return '';
})
.filter(name => name !== '');
// Create specific guidance based on the error and tool
let guidance = `TOOL GUIDANCE: The tool '${toolName}' failed with error: ${errorMessage}.\n`;
// Add suggestions based on the specific tool and error
if (toolName === 'attribute_search' && errorMessage.includes('Invalid attribute type')) {
guidance += "CORRECT USAGE: The 'attribute_search' tool requires a valid 'attributeType' parameter that must be either 'label' or 'relation'.\n";
guidance += "EXAMPLE: { \"attributeType\": \"label\", \"attributeValue\": \"important\" }\n";
}
else if (errorMessage.includes('Tool not found')) {
// Provide guidance on available search tools if a tool wasn't found
const searchTools = availableToolNames.filter(name => name.includes('search'));
guidance += `AVAILABLE SEARCH TOOLS: ${searchTools.join(', ')}\n`;
guidance += "TRY VECTOR SEARCH: For conceptual matches, use 'vector_search' with a query parameter.\n";
guidance += "EXAMPLE: { \"query\": \"your search terms here\" }\n";
}
else if (errorMessage.includes('missing required parameter')) {
// Provide parameter guidance based on the tool name
if (toolName === 'vector_search') {
guidance += "REQUIRED PARAMETERS: The 'vector_search' tool requires a 'query' parameter.\n";
guidance += "EXAMPLE: { \"query\": \"your search terms here\" }\n";
} else if (toolName === 'keyword_search') {
guidance += "REQUIRED PARAMETERS: The 'keyword_search' tool requires a 'query' parameter.\n";
guidance += "EXAMPLE: { \"query\": \"your search terms here\" }\n";
}
}
// Add a general suggestion to try vector_search as a fallback
if (!toolName.includes('vector_search')) {
guidance += "RECOMMENDATION: If specific searches fail, try the 'vector_search' tool which performs semantic searches.\n";
}
return guidance;
}
/**
* Determines if a tool result is effectively empty or unhelpful
* @param result The result from the tool execution