mirror of
https://github.com/TriliumNext/Notes.git
synced 2025-07-29 19:12:27 +08:00
948 lines
42 KiB
TypeScript
948 lines
42 KiB
TypeScript
import log from '../../../log.js';
|
|
import providerManager from './provider_manager.js';
|
|
import cacheManager from './cache_manager.js';
|
|
import semanticSearch from './semantic_search.js';
|
|
import queryEnhancer from './query_enhancer.js';
|
|
import contextFormatter from './context_formatter.js';
|
|
import aiServiceManager from '../../ai_service_manager.js';
|
|
import { ContextExtractor } from '../index.js';
|
|
import { CONTEXT_PROMPTS } from '../../constants/llm_prompt_constants.js';
|
|
import becca from '../../../../becca/becca.js';
|
|
import type { NoteSearchResult } from '../../interfaces/context_interfaces.js';
|
|
import type { LLMServiceInterface } from '../../interfaces/agent_tool_interfaces.js';
|
|
import type { Message } from '../../ai_interface.js';
|
|
|
|
/**
|
|
* Main context service that integrates all context-related functionality
|
|
* This service replaces the old TriliumContextService and SemanticContextService
|
|
*/
|
|
export class ContextService {
|
|
private initialized = false;
|
|
private initPromise: Promise<void> | null = null;
|
|
private contextExtractor: ContextExtractor;
|
|
|
|
constructor() {
|
|
this.contextExtractor = new ContextExtractor();
|
|
}
|
|
|
|
/**
|
|
* Initialize the service
|
|
*/
|
|
async initialize(): Promise<void> {
|
|
if (this.initialized) return;
|
|
|
|
// Use a promise to prevent multiple simultaneous initializations
|
|
if (this.initPromise) return this.initPromise;
|
|
|
|
this.initPromise = (async () => {
|
|
try {
|
|
// Initialize provider
|
|
const provider = await providerManager.getPreferredEmbeddingProvider();
|
|
if (!provider) {
|
|
throw new Error(`No embedding provider available. Could not initialize context service.`);
|
|
}
|
|
|
|
// Initialize agent tools to ensure they're ready
|
|
try {
|
|
await aiServiceManager.getInstance().initializeAgentTools();
|
|
log.info("Agent tools initialized for use with ContextService");
|
|
} catch (toolError) {
|
|
log.error(`Error initializing agent tools: ${toolError}`);
|
|
// Continue even if agent tools fail to initialize
|
|
}
|
|
|
|
this.initialized = true;
|
|
log.info(`Context service initialized with provider: ${provider.name}`);
|
|
} catch (error: unknown) {
|
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
log.error(`Failed to initialize context service: ${errorMessage}`);
|
|
throw error;
|
|
} finally {
|
|
this.initPromise = null;
|
|
}
|
|
})();
|
|
|
|
return this.initPromise;
|
|
}
|
|
|
|
/**
|
|
* Process a user query to find relevant context in Trilium notes
|
|
*
|
|
* @param userQuestion - The user's query
|
|
* @param llmService - The LLM service to use
|
|
* @param contextNoteId - Optional note ID to restrict search to a branch
|
|
* @param showThinking - Whether to show the thinking process in output
|
|
* @returns Context information and relevant notes
|
|
*/
|
|
async processQuery(
|
|
userQuestion: string,
|
|
llmService: LLMServiceInterface,
|
|
contextNoteId: string | null = null,
|
|
showThinking: boolean = false
|
|
): Promise<{ context: string; sources: NoteSearchResult[]; thinking?: string }> {
|
|
log.info(`Processing query with: question="${userQuestion.substring(0, 50)}...", noteId=${contextNoteId}, showThinking=${showThinking}`);
|
|
|
|
if (!this.initialized) {
|
|
try {
|
|
await this.initialize();
|
|
} catch (error) {
|
|
log.error(`Failed to initialize ContextService: ${error}`);
|
|
// Return a fallback response if initialization fails
|
|
return {
|
|
context: CONTEXT_PROMPTS.NO_NOTES_CONTEXT,
|
|
sources: [],
|
|
thinking: undefined
|
|
};
|
|
}
|
|
}
|
|
|
|
try {
|
|
// Step 1: Generate search queries
|
|
let searchQueries: string[];
|
|
try {
|
|
searchQueries = await queryEnhancer.generateSearchQueries(userQuestion, llmService);
|
|
} catch (error) {
|
|
log.error(`Error generating search queries, using fallback: ${error}`);
|
|
searchQueries = [userQuestion]; // Fallback to using the original question
|
|
}
|
|
log.info(`Generated search queries: ${JSON.stringify(searchQueries)}`);
|
|
|
|
// Step 2: Find relevant notes using multi-query approach
|
|
let relevantNotes: NoteSearchResult[] = [];
|
|
try {
|
|
// Find notes for each query and combine results
|
|
const allResults: Map<string, NoteSearchResult> = new Map();
|
|
|
|
for (const query of searchQueries) {
|
|
const results = await semanticSearch.findRelevantNotes(
|
|
query,
|
|
contextNoteId,
|
|
5 // Limit per query
|
|
);
|
|
|
|
// Combine results, avoiding duplicates
|
|
for (const result of results) {
|
|
if (!allResults.has(result.noteId)) {
|
|
allResults.set(result.noteId, result);
|
|
} else {
|
|
// If note already exists, update similarity to max of both values
|
|
const existing = allResults.get(result.noteId);
|
|
if (existing && result.similarity > existing.similarity) {
|
|
existing.similarity = result.similarity;
|
|
allResults.set(result.noteId, existing);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Convert map to array and limit to top results
|
|
relevantNotes = Array.from(allResults.values())
|
|
.filter(note => {
|
|
// Filter out notes with no content or very minimal content (less than 10 chars)
|
|
const hasContent = note.content && note.content.trim().length > 10;
|
|
if (!hasContent) {
|
|
log.info(`Filtering out empty/minimal note: "${note.title}" (${note.noteId})`);
|
|
}
|
|
return hasContent;
|
|
})
|
|
.sort((a, b) => b.similarity - a.similarity)
|
|
.slice(0, 20); // Increased from 8 to 20 notes
|
|
|
|
log.info(`After filtering out empty notes, ${relevantNotes.length} relevant notes remain`);
|
|
} catch (error) {
|
|
log.error(`Error finding relevant notes: ${error}`);
|
|
// Continue with empty notes list
|
|
}
|
|
|
|
// Step 3: Build context from the notes
|
|
const provider = await providerManager.getPreferredEmbeddingProvider();
|
|
const providerId = provider?.name || 'default';
|
|
const context = await contextFormatter.buildContextFromNotes(relevantNotes, userQuestion, providerId);
|
|
|
|
// DEBUG: Log the initial context built from notes
|
|
log.info(`Initial context from buildContextFromNotes: ${context.length} chars, starting with: "${context.substring(0, 150)}..."`);
|
|
|
|
// Step 4: Add agent tools context with thinking process if requested
|
|
let enhancedContext = context;
|
|
try {
|
|
// Pass 'root' as the default noteId when no specific note is selected
|
|
const noteIdToUse = contextNoteId || 'root';
|
|
log.info(`Calling getAgentToolsContext with noteId=${noteIdToUse}, showThinking=${showThinking}`);
|
|
|
|
const agentContext = await this.getAgentToolsContext(
|
|
noteIdToUse,
|
|
userQuestion,
|
|
showThinking,
|
|
relevantNotes
|
|
);
|
|
|
|
if (agentContext) {
|
|
enhancedContext = enhancedContext + "\n\n" + agentContext;
|
|
}
|
|
|
|
// DEBUG: Log the final combined context
|
|
log.info(`FINAL COMBINED CONTEXT: ${enhancedContext.length} chars, with content structure: ${this.summarizeContextStructure(enhancedContext)}`);
|
|
} catch (error) {
|
|
log.error(`Error getting agent tools context: ${error}`);
|
|
// Continue with the basic context
|
|
}
|
|
|
|
return {
|
|
context: enhancedContext,
|
|
sources: relevantNotes,
|
|
thinking: showThinking ? this.summarizeContextStructure(enhancedContext) : undefined
|
|
};
|
|
} catch (error) {
|
|
log.error(`Error processing query: ${error}`);
|
|
return {
|
|
context: CONTEXT_PROMPTS.NO_NOTES_CONTEXT,
|
|
sources: [],
|
|
thinking: undefined
|
|
};
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get context with agent tools enhancement
|
|
*
|
|
* @param noteId - The relevant note ID
|
|
* @param query - The user's query
|
|
* @param showThinking - Whether to show thinking process
|
|
* @param relevantNotes - Optional pre-found relevant notes
|
|
* @returns Enhanced context string
|
|
*/
|
|
async getAgentToolsContext(
|
|
noteId: string,
|
|
query: string,
|
|
showThinking: boolean = false,
|
|
relevantNotes: NoteSearchResult[] = []
|
|
): Promise<string> {
|
|
try {
|
|
log.info(`Building enhanced agent tools context for query: "${query.substring(0, 50)}...", noteId=${noteId}, showThinking=${showThinking}`);
|
|
|
|
// Make sure agent tools are initialized
|
|
const agentManager = aiServiceManager.getInstance();
|
|
|
|
// Initialize all tools if not already done
|
|
if (!agentManager.getAgentTools().isInitialized()) {
|
|
await agentManager.initializeAgentTools();
|
|
log.info("Agent tools initialized on-demand in getAgentToolsContext");
|
|
}
|
|
|
|
// Get all agent tools
|
|
const vectorSearchTool = agentManager.getVectorSearchTool();
|
|
const noteNavigatorTool = agentManager.getNoteNavigatorTool();
|
|
const queryDecompositionTool = agentManager.getQueryDecompositionTool();
|
|
const contextualThinkingTool = agentManager.getContextualThinkingTool();
|
|
|
|
// Step 1: Start a thinking process
|
|
const thinkingId = contextualThinkingTool.startThinking(query);
|
|
contextualThinkingTool.addThinkingStep(thinkingId, {
|
|
type: 'observation',
|
|
content: `Analyzing query: "${query}" for note ID: ${noteId}`
|
|
});
|
|
|
|
// Step 2: Decompose the query into sub-questions
|
|
const decomposedQuery = queryDecompositionTool.decomposeQuery(query);
|
|
contextualThinkingTool.addThinkingStep(thinkingId, {
|
|
type: 'observation',
|
|
content: `Query complexity: ${decomposedQuery.complexity}/10. Decomposed into ${decomposedQuery.subQueries.length} sub-queries.`
|
|
});
|
|
|
|
// Log each sub-query as a thinking step
|
|
for (const subQuery of decomposedQuery.subQueries) {
|
|
contextualThinkingTool.addThinkingStep(thinkingId, {
|
|
type: 'question',
|
|
content: subQuery.text,
|
|
metadata: {
|
|
reason: subQuery.reason
|
|
}
|
|
});
|
|
}
|
|
|
|
// Step 3: Use vector search to find related content
|
|
// Use an aggressive search with lower threshold to get more results
|
|
const searchOptions = {
|
|
threshold: 0.5, // Lower threshold to include more matches
|
|
limit: 15 // Get more results
|
|
};
|
|
|
|
const vectorSearchPromises = [];
|
|
|
|
// Search for each sub-query that isn't just the original query
|
|
for (const subQuery of decomposedQuery.subQueries.filter(sq => sq.text !== query)) {
|
|
vectorSearchPromises.push(
|
|
vectorSearchTool.search(subQuery.text, noteId, searchOptions)
|
|
.then(results => {
|
|
return {
|
|
query: subQuery.text,
|
|
results
|
|
};
|
|
})
|
|
);
|
|
}
|
|
|
|
// Wait for all searches to complete
|
|
const searchResults = await Promise.all(vectorSearchPromises);
|
|
|
|
// Record the search results in thinking steps
|
|
let totalResults = 0;
|
|
for (const result of searchResults) {
|
|
totalResults += result.results.length;
|
|
|
|
if (result.results.length > 0) {
|
|
const stepId = contextualThinkingTool.addThinkingStep(thinkingId, {
|
|
type: 'evidence',
|
|
content: `Found ${result.results.length} relevant notes for sub-query: "${result.query}"`,
|
|
metadata: {
|
|
searchQuery: result.query
|
|
}
|
|
});
|
|
|
|
// Add top results as children
|
|
for (const note of result.results.slice(0, 3)) {
|
|
contextualThinkingTool.addThinkingStep(thinkingId, {
|
|
type: 'evidence',
|
|
content: `Note "${note.title}" (similarity: ${Math.round(note.similarity * 100)}%) contains relevant information`,
|
|
metadata: {
|
|
noteId: note.noteId,
|
|
similarity: note.similarity
|
|
}
|
|
}, stepId);
|
|
}
|
|
} else {
|
|
contextualThinkingTool.addThinkingStep(thinkingId, {
|
|
type: 'observation',
|
|
content: `No notes found for sub-query: "${result.query}"`,
|
|
metadata: {
|
|
searchQuery: result.query
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
// Step 4: Get note structure information
|
|
try {
|
|
const noteStructure = await noteNavigatorTool.getNoteStructure(noteId);
|
|
|
|
contextualThinkingTool.addThinkingStep(thinkingId, {
|
|
type: 'observation',
|
|
content: `Note structure: ${noteStructure.childCount} child notes, ${noteStructure.attributes.length} attributes, ${noteStructure.parentPath.length} levels in hierarchy`,
|
|
metadata: {
|
|
structure: noteStructure
|
|
}
|
|
});
|
|
|
|
// Add information about parent path
|
|
if (noteStructure.parentPath.length > 0) {
|
|
const parentPathStr = noteStructure.parentPath.map((p: {title: string, noteId: string}) => p.title).join(' > ');
|
|
contextualThinkingTool.addThinkingStep(thinkingId, {
|
|
type: 'observation',
|
|
content: `Note hierarchy: ${parentPathStr}`,
|
|
metadata: {
|
|
parentPath: noteStructure.parentPath
|
|
}
|
|
});
|
|
}
|
|
} catch (error) {
|
|
log.error(`Error getting note structure: ${error}`);
|
|
contextualThinkingTool.addThinkingStep(thinkingId, {
|
|
type: 'observation',
|
|
content: `Unable to retrieve note structure information: ${error}`
|
|
});
|
|
}
|
|
|
|
// Step 5: Conclude thinking process
|
|
contextualThinkingTool.addThinkingStep(thinkingId, {
|
|
type: 'conclusion',
|
|
content: `Analysis complete. Found ${totalResults} relevant notes across ${searchResults.length} search queries.`,
|
|
metadata: {
|
|
totalResults,
|
|
queryCount: searchResults.length
|
|
}
|
|
});
|
|
|
|
// Complete the thinking process
|
|
contextualThinkingTool.completeThinking(thinkingId);
|
|
|
|
// Step 6: Build the context string combining all the information
|
|
let agentContext = '';
|
|
|
|
// Add note structure information
|
|
try {
|
|
const noteStructure = await noteNavigatorTool.getNoteStructure(noteId);
|
|
agentContext += `## Current Note Context\n`;
|
|
agentContext += `- Note Title: ${noteStructure.title}\n`;
|
|
|
|
if (noteStructure.parentPath.length > 0) {
|
|
const parentPathStr = noteStructure.parentPath.map((p: {title: string, noteId: string}) => p.title).join(' > ');
|
|
agentContext += `- Location: ${parentPathStr}\n`;
|
|
}
|
|
|
|
if (noteStructure.attributes.length > 0) {
|
|
agentContext += `- Attributes: ${noteStructure.attributes.map((a: {name: string, value: string}) => `${a.name}=${a.value}`).join(', ')}\n`;
|
|
}
|
|
|
|
if (noteStructure.childCount > 0) {
|
|
agentContext += `- Contains ${noteStructure.childCount} child notes\n`;
|
|
}
|
|
|
|
agentContext += `\n`;
|
|
} catch (error) {
|
|
log.error(`Error adding note structure to context: ${error}`);
|
|
}
|
|
|
|
// Combine the notes from both searches - the initial relevantNotes and from vector search
|
|
// Start with a Map to deduplicate by noteId
|
|
const allNotes = new Map<string, NoteSearchResult>();
|
|
|
|
// Add notes from the initial search in processQuery (relevantNotes parameter)
|
|
if (relevantNotes && relevantNotes.length > 0) {
|
|
log.info(`Adding ${relevantNotes.length} notes from initial search to combined results`);
|
|
for (const note of relevantNotes) {
|
|
if (note.noteId) {
|
|
allNotes.set(note.noteId, note);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Add notes from vector search of sub-queries
|
|
const vectorSearchNotes = searchResults.flatMap(r => r.results);
|
|
if (vectorSearchNotes.length > 0) {
|
|
log.info(`Adding ${vectorSearchNotes.length} notes from vector search to combined results`);
|
|
for (const note of vectorSearchNotes) {
|
|
// If note already exists, keep the one with higher similarity
|
|
const existing = allNotes.get(note.noteId);
|
|
if (existing && note.similarity > existing.similarity) {
|
|
existing.similarity = note.similarity;
|
|
} else {
|
|
allNotes.set(note.noteId, note);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Convert the combined Map to an array and sort by similarity
|
|
const combinedNotes = Array.from(allNotes.values())
|
|
.filter(note => {
|
|
// Filter out notes with no content or very minimal content
|
|
const hasContent = note.content && note.content.trim().length > 10;
|
|
if (!hasContent) {
|
|
log.info(`Filtering out empty/minimal note from combined results: "${note.title}" (${note.noteId})`);
|
|
}
|
|
return hasContent;
|
|
})
|
|
.sort((a, b) => b.similarity - a.similarity);
|
|
|
|
log.info(`Combined ${relevantNotes.length} notes from initial search with ${vectorSearchNotes.length} notes from vector search, resulting in ${combinedNotes.length} unique notes after filtering out empty notes`);
|
|
|
|
// Filter for Qu-related notes
|
|
const quNotes = combinedNotes.filter(result =>
|
|
result.title.toLowerCase().includes('qu') ||
|
|
(result.content && result.content.toLowerCase().includes('qu'))
|
|
);
|
|
|
|
if (quNotes.length > 0) {
|
|
log.info(`Found ${quNotes.length} Qu-related notes out of ${combinedNotes.length} total notes`);
|
|
quNotes.forEach((note, idx) => {
|
|
if (idx < 3) { // Log just a sample to avoid log spam
|
|
log.info(`Qu note ${idx+1}: "${note.title}" (similarity: ${Math.round(note.similarity * 100)}%), content length: ${note.content ? note.content.length : 0} chars`);
|
|
}
|
|
});
|
|
|
|
// Prioritize Qu notes first, then other notes by similarity
|
|
const nonQuNotes = combinedNotes.filter(note => !quNotes.includes(note));
|
|
const finalNotes = [...quNotes, ...nonQuNotes].slice(0, 30); // Take top 30 prioritized notes
|
|
|
|
log.info(`Selected ${finalNotes.length} notes for context, with ${quNotes.length} Qu-related notes prioritized`);
|
|
|
|
// Add the selected notes to the context
|
|
if (finalNotes.length > 0) {
|
|
agentContext += `## Relevant Information\n`;
|
|
|
|
for (const note of finalNotes) {
|
|
agentContext += `### ${note.title}\n`;
|
|
|
|
// Add relationship information for the note
|
|
try {
|
|
const noteObj = becca.getNote(note.noteId);
|
|
if (noteObj) {
|
|
// Get parent notes
|
|
const parentNotes = noteObj.getParentNotes();
|
|
if (parentNotes && parentNotes.length > 0) {
|
|
agentContext += `**Parent notes:** ${parentNotes.map((p: any) => p.title).join(', ')}\n`;
|
|
}
|
|
|
|
// Get child notes
|
|
const childNotes = noteObj.getChildNotes();
|
|
if (childNotes && childNotes.length > 0) {
|
|
agentContext += `**Child notes:** ${childNotes.map((c: any) => c.title).join(', ')}\n`;
|
|
}
|
|
|
|
// Get attributes
|
|
const attributes = noteObj.getAttributes();
|
|
if (attributes && attributes.length > 0) {
|
|
const filteredAttrs = attributes.filter((a: any) => !a.name.startsWith('_')); // Filter out system attributes
|
|
if (filteredAttrs.length > 0) {
|
|
agentContext += `**Attributes:** ${filteredAttrs.map((a: any) => `${a.name}=${a.value}`).join(', ')}\n`;
|
|
}
|
|
}
|
|
|
|
// Get backlinks/related notes through relation attributes
|
|
const relationAttrs = attributes?.filter((a: any) =>
|
|
a.name.startsWith('relation:') ||
|
|
a.name.startsWith('label:')
|
|
);
|
|
|
|
if (relationAttrs && relationAttrs.length > 0) {
|
|
agentContext += `**Relationships:** ${relationAttrs.map((a: any) => {
|
|
const targetNote = becca.getNote(a.value);
|
|
const targetTitle = targetNote ? targetNote.title : a.value;
|
|
return `${a.name.substring(a.name.indexOf(':') + 1)} → ${targetTitle}`;
|
|
}).join(', ')}\n`;
|
|
}
|
|
}
|
|
} catch (error) {
|
|
log.error(`Error getting relationship info for note ${note.noteId}: ${error}`);
|
|
}
|
|
|
|
agentContext += '\n';
|
|
|
|
if (note.content) {
|
|
// Extract relevant content instead of just taking first 2000 chars
|
|
const relevantContent = await this.extractRelevantContent(note.content, query, 2000);
|
|
agentContext += `${relevantContent}\n\n`;
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
log.info(`No Qu-related notes found among the ${combinedNotes.length} combined notes`);
|
|
|
|
// Just take the top notes by similarity
|
|
const finalNotes = combinedNotes.slice(0, 30); // Take top 30 notes
|
|
|
|
if (finalNotes.length > 0) {
|
|
agentContext += `## Relevant Information\n`;
|
|
|
|
for (const note of finalNotes) {
|
|
agentContext += `### ${note.title}\n`;
|
|
|
|
// Add relationship information for the note
|
|
try {
|
|
const noteObj = becca.getNote(note.noteId);
|
|
if (noteObj) {
|
|
// Get parent notes
|
|
const parentNotes = noteObj.getParentNotes();
|
|
if (parentNotes && parentNotes.length > 0) {
|
|
agentContext += `**Parent notes:** ${parentNotes.map((p: any) => p.title).join(', ')}\n`;
|
|
}
|
|
|
|
// Get child notes
|
|
const childNotes = noteObj.getChildNotes();
|
|
if (childNotes && childNotes.length > 0) {
|
|
agentContext += `**Child notes:** ${childNotes.map((c: any) => c.title).join(', ')}\n`;
|
|
}
|
|
|
|
// Get attributes
|
|
const attributes = noteObj.getAttributes();
|
|
if (attributes && attributes.length > 0) {
|
|
const filteredAttrs = attributes.filter((a: any) => !a.name.startsWith('_')); // Filter out system attributes
|
|
if (filteredAttrs.length > 0) {
|
|
agentContext += `**Attributes:** ${filteredAttrs.map((a: any) => `${a.name}=${a.value}`).join(', ')}\n`;
|
|
}
|
|
}
|
|
|
|
// Get backlinks/related notes through relation attributes
|
|
const relationAttrs = attributes?.filter((a: any) =>
|
|
a.name.startsWith('relation:') ||
|
|
a.name.startsWith('label:')
|
|
);
|
|
|
|
if (relationAttrs && relationAttrs.length > 0) {
|
|
agentContext += `**Relationships:** ${relationAttrs.map((a: any) => {
|
|
const targetNote = becca.getNote(a.value);
|
|
const targetTitle = targetNote ? targetNote.title : a.value;
|
|
return `${a.name.substring(a.name.indexOf(':') + 1)} → ${targetTitle}`;
|
|
}).join(', ')}\n`;
|
|
}
|
|
}
|
|
} catch (error) {
|
|
log.error(`Error getting relationship info for note ${note.noteId}: ${error}`);
|
|
}
|
|
|
|
agentContext += '\n';
|
|
|
|
if (note.content) {
|
|
// Extract relevant content instead of just taking first 2000 chars
|
|
const relevantContent = await this.extractRelevantContent(note.content, query, 2000);
|
|
agentContext += `${relevantContent}\n\n`;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Add thinking process if requested
|
|
if (showThinking) {
|
|
log.info(`Including thinking process in context (showThinking=true)`);
|
|
agentContext += `\n## Reasoning Process\n`;
|
|
const thinkingSummary = contextualThinkingTool.getThinkingSummary(thinkingId);
|
|
log.info(`Thinking summary length: ${thinkingSummary.length} characters`);
|
|
agentContext += thinkingSummary;
|
|
} else {
|
|
log.info(`Skipping thinking process in context (showThinking=false)`);
|
|
}
|
|
|
|
// Log stats about the context
|
|
log.info(`Agent tools context built: ${agentContext.length} chars, ${agentContext.split('\n').length} lines`);
|
|
|
|
// DEBUG: Log more detailed information about the agent tools context content
|
|
log.info(`Agent tools context content structure: ${this.summarizeContextStructure(agentContext)}`);
|
|
if (agentContext.length < 1000) {
|
|
log.info(`Agent tools context full content (short): ${agentContext}`);
|
|
} else {
|
|
log.info(`Agent tools context first 500 chars: ${agentContext.substring(0, 500)}...`);
|
|
log.info(`Agent tools context last 500 chars: ${agentContext.substring(agentContext.length - 500)}`);
|
|
}
|
|
|
|
return agentContext;
|
|
} catch (error) {
|
|
log.error(`Error getting agent tools context: ${error}`);
|
|
return `Error generating enhanced context: ${error}`;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Summarize the structure of a context string for debugging
|
|
* @param context - The context string to summarize
|
|
* @returns A summary of the context structure
|
|
*/
|
|
private summarizeContextStructure(context: string): string {
|
|
if (!context) return "Empty context";
|
|
|
|
// Count sections and headers
|
|
const sections = context.split('##').length - 1;
|
|
const subSections = context.split('###').length - 1;
|
|
|
|
// Count notes referenced
|
|
const noteMatches = context.match(/### [^\n]+/g);
|
|
const noteCount = noteMatches ? noteMatches.length : 0;
|
|
|
|
// Extract note titles if present
|
|
let noteTitles = "";
|
|
if (noteMatches && noteMatches.length > 0) {
|
|
noteTitles = ` Note titles: ${noteMatches.slice(0, 3).map(m => m.substring(4)).join(', ')}${noteMatches.length > 3 ? '...' : ''}`;
|
|
}
|
|
|
|
return `${sections} main sections, ${subSections} subsections, ${noteCount} notes referenced.${noteTitles}`;
|
|
}
|
|
|
|
/**
|
|
* Get semantic context based on query
|
|
*
|
|
* @param noteId - Note ID to start from
|
|
* @param userQuery - User query for context
|
|
* @param maxResults - Maximum number of results
|
|
* @param messages - Optional conversation messages to adjust context size
|
|
* @returns Formatted context
|
|
*/
|
|
async getSemanticContext(
|
|
noteId: string,
|
|
userQuery: string,
|
|
maxResults: number = 5,
|
|
messages: Message[] = []
|
|
): Promise<string> {
|
|
if (!this.initialized) {
|
|
await this.initialize();
|
|
}
|
|
|
|
try {
|
|
// Get related notes from the context extractor
|
|
const [
|
|
parentNotes,
|
|
childNotes,
|
|
linkedNotes
|
|
] = await Promise.all([
|
|
this.contextExtractor.getParentNotes(noteId, 3),
|
|
this.contextExtractor.getChildContext(noteId, 10).then(context => {
|
|
// Parse child notes from context string
|
|
const lines = context.split('\n');
|
|
const result: {noteId: string, title: string}[] = [];
|
|
for (const line of lines) {
|
|
const match = line.match(/- (.*)/);
|
|
if (match) {
|
|
// We don't have noteIds in the context string, so use titles only
|
|
result.push({
|
|
title: match[1],
|
|
noteId: '' // Empty noteId since we can't extract it from context
|
|
});
|
|
}
|
|
}
|
|
return result;
|
|
}),
|
|
this.contextExtractor.getLinkedNotesContext(noteId, 10).then(context => {
|
|
// Parse linked notes from context string
|
|
const lines = context.split('\n');
|
|
const result: {noteId: string, title: string}[] = [];
|
|
for (const line of lines) {
|
|
const match = line.match(/- \[(.*?)\]\(trilium:\/\/([a-zA-Z0-9]+)\)/);
|
|
if (match) {
|
|
result.push({
|
|
title: match[1],
|
|
noteId: match[2]
|
|
});
|
|
}
|
|
}
|
|
return result;
|
|
})
|
|
]);
|
|
|
|
// Combine all related notes
|
|
const allRelatedNotes = [...parentNotes, ...childNotes, ...linkedNotes];
|
|
|
|
// If no related notes, return empty context
|
|
if (allRelatedNotes.length === 0) {
|
|
return '';
|
|
}
|
|
|
|
// Convert parent notes from {id, title} to {noteId, title} for consistency
|
|
const normalizedRelatedNotes = allRelatedNotes.map(note => {
|
|
return {
|
|
noteId: 'id' in note ? note.id : note.noteId,
|
|
title: note.title
|
|
};
|
|
});
|
|
|
|
// Rank notes by relevance to query
|
|
const rankedNotes = await semanticSearch.rankNotesByRelevance(
|
|
normalizedRelatedNotes as Array<{noteId: string, title: string}>,
|
|
userQuery
|
|
);
|
|
|
|
// Get content for the top N most relevant notes
|
|
const mostRelevantNotes = rankedNotes.slice(0, maxResults);
|
|
|
|
// Get relevant search results to pass to context formatter
|
|
const searchResults = await Promise.all(
|
|
mostRelevantNotes.map(async note => {
|
|
const content = await this.contextExtractor.getNoteContent(note.noteId);
|
|
if (!content) return null;
|
|
|
|
// Create a properly typed NoteSearchResult object
|
|
return {
|
|
noteId: note.noteId,
|
|
title: note.title,
|
|
content,
|
|
similarity: note.relevance
|
|
};
|
|
})
|
|
);
|
|
|
|
// Filter out nulls and empty content
|
|
const validResults: NoteSearchResult[] = searchResults
|
|
.filter(result => result !== null && result.content && result.content.trim().length > 0)
|
|
.map(result => result as NoteSearchResult);
|
|
|
|
// If no content retrieved, return empty string
|
|
if (validResults.length === 0) {
|
|
return '';
|
|
}
|
|
|
|
// Get the provider information for formatting
|
|
const provider = await providerManager.getPreferredEmbeddingProvider();
|
|
const providerId = provider?.name || 'default';
|
|
|
|
// Format the context with the context formatter (which handles adjusting for conversation size)
|
|
return contextFormatter.buildContextFromNotes(validResults, userQuery, providerId, messages);
|
|
} catch (error) {
|
|
log.error(`Error getting semantic context: ${error}`);
|
|
return '';
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get progressive context loading based on depth
|
|
*
|
|
* @param noteId - The base note ID
|
|
* @param depth - Depth level (1-4)
|
|
* @returns Context string with progressively more information
|
|
*/
|
|
async getProgressiveContext(noteId: string, depth: number = 1): Promise<string> {
|
|
if (!this.initialized) {
|
|
await this.initialize();
|
|
}
|
|
|
|
try {
|
|
// Use the existing context extractor method
|
|
return await this.contextExtractor.getProgressiveContext(noteId, depth);
|
|
} catch (error) {
|
|
log.error(`Error getting progressive context: ${error}`);
|
|
return '';
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get smart context that adapts to query complexity
|
|
*
|
|
* @param noteId - The base note ID
|
|
* @param userQuery - The user's query
|
|
* @returns Context string with appropriate level of detail
|
|
*/
|
|
async getSmartContext(noteId: string, userQuery: string): Promise<string> {
|
|
if (!this.initialized) {
|
|
await this.initialize();
|
|
}
|
|
|
|
try {
|
|
// Determine query complexity to adjust context depth
|
|
const complexity = queryEnhancer.estimateQueryComplexity(userQuery);
|
|
|
|
// If it's a simple query with low complexity, use progressive context
|
|
if (complexity < 0.3) {
|
|
return await this.getProgressiveContext(noteId, 2); // Just note + parents
|
|
}
|
|
// For medium complexity, include more context
|
|
else if (complexity < 0.7) {
|
|
return await this.getProgressiveContext(noteId, 3); // Note + parents + children
|
|
}
|
|
// For complex queries, use semantic context
|
|
else {
|
|
return await this.getSemanticContext(noteId, userQuery, 7); // More results for complex queries
|
|
}
|
|
} catch (error) {
|
|
log.error(`Error getting smart context: ${error}`);
|
|
// Fallback to basic context extraction
|
|
return await this.contextExtractor.extractContext(noteId);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Clear all context caches
|
|
*/
|
|
clearCaches(): void {
|
|
cacheManager.clearAllCaches();
|
|
}
|
|
|
|
/**
|
|
* Extract the most relevant portions from a note's content
|
|
* @param content - The full note content
|
|
* @param query - The user's query
|
|
* @param maxChars - Maximum characters to include
|
|
* @returns The most relevant content sections
|
|
*/
|
|
private async extractRelevantContent(content: string, query: string, maxChars: number = 2000): Promise<string> {
|
|
if (!content || content.length <= maxChars) {
|
|
return content; // Return full content if it's already short enough
|
|
}
|
|
|
|
try {
|
|
// Get the vector search tool for relevance calculation
|
|
const agentManager = aiServiceManager.getInstance();
|
|
const vectorSearchTool = agentManager.getVectorSearchTool();
|
|
|
|
// Split content into chunks of reasonable size (300-500 chars with overlap)
|
|
const chunkSize = 400;
|
|
const overlap = 100;
|
|
const chunks: string[] = [];
|
|
|
|
for (let i = 0; i < content.length; i += (chunkSize - overlap)) {
|
|
const end = Math.min(i + chunkSize, content.length);
|
|
chunks.push(content.substring(i, end));
|
|
if (end === content.length) break;
|
|
}
|
|
|
|
log.info(`Split note content into ${chunks.length} chunks for relevance extraction`);
|
|
|
|
// Get embedding provider from service
|
|
const provider = await providerManager.getPreferredEmbeddingProvider();
|
|
if (!provider) {
|
|
throw new Error("No embedding provider available");
|
|
}
|
|
|
|
// Get embeddings for the query and all chunks
|
|
const queryEmbedding = await provider.generateEmbeddings(query);
|
|
|
|
// Process chunks in smaller batches to avoid overwhelming the provider
|
|
const batchSize = 5;
|
|
const chunkEmbeddings = [];
|
|
|
|
for (let i = 0; i < chunks.length; i += batchSize) {
|
|
const batch = chunks.slice(i, i + batchSize);
|
|
const batchEmbeddings = await Promise.all(
|
|
batch.map(chunk => provider.generateEmbeddings(chunk))
|
|
);
|
|
chunkEmbeddings.push(...batchEmbeddings);
|
|
}
|
|
|
|
// Calculate similarity between query and each chunk
|
|
const similarities: Array<{index: number, similarity: number, content: string}> =
|
|
chunkEmbeddings.map((embedding, index) => {
|
|
// Calculate cosine similarity manually since the method doesn't exist
|
|
const similarity = this.calculateCosineSimilarity(queryEmbedding, embedding);
|
|
return { index, similarity, content: chunks[index] };
|
|
});
|
|
|
|
// Sort chunks by similarity (most relevant first)
|
|
similarities.sort((a, b) => b.similarity - a.similarity);
|
|
|
|
// DEBUG: Log some info about the top chunks
|
|
log.info(`Top 3 most relevant chunks for query "${query.substring(0, 30)}..." (out of ${chunks.length} total):`);
|
|
similarities.slice(0, 3).forEach((chunk, idx) => {
|
|
log.info(` Chunk ${idx+1}: Similarity ${Math.round(chunk.similarity * 100)}%, Content: "${chunk.content.substring(0, 50)}..."`);
|
|
});
|
|
|
|
// Take the most relevant chunks up to maxChars
|
|
let result = '';
|
|
let totalChars = 0;
|
|
let chunksIncluded = 0;
|
|
|
|
for (const chunk of similarities) {
|
|
if (totalChars + chunk.content.length > maxChars) {
|
|
// If adding full chunk would exceed limit, add as much as possible
|
|
const remainingSpace = maxChars - totalChars;
|
|
if (remainingSpace > 100) { // Only add if we can include something meaningful
|
|
result += `\n...\n${chunk.content.substring(0, remainingSpace)}...`;
|
|
log.info(` Added partial chunk with similarity ${Math.round(chunk.similarity * 100)}% (${remainingSpace} chars)`);
|
|
}
|
|
break;
|
|
}
|
|
|
|
if (result.length > 0) result += '\n...\n';
|
|
result += chunk.content;
|
|
totalChars += chunk.content.length;
|
|
chunksIncluded++;
|
|
}
|
|
|
|
log.info(`Extracted ${totalChars} chars of relevant content from ${content.length} chars total (${chunksIncluded} chunks included)`);
|
|
return result;
|
|
} catch (error) {
|
|
log.error(`Error extracting relevant content: ${error}`);
|
|
// Fallback to simple truncation if extraction fails
|
|
return content.substring(0, maxChars) + '...';
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Calculate cosine similarity between two vectors
|
|
* @param vec1 - First vector
|
|
* @param vec2 - Second vector
|
|
* @returns Cosine similarity between the two vectors
|
|
*/
|
|
private calculateCosineSimilarity(vec1: number[], vec2: number[]): number {
|
|
let dotProduct = 0;
|
|
let norm1 = 0;
|
|
let norm2 = 0;
|
|
|
|
for (let i = 0; i < vec1.length; i++) {
|
|
dotProduct += vec1[i] * vec2[i];
|
|
norm1 += vec1[i] * vec1[i];
|
|
norm2 += vec2[i] * vec2[i];
|
|
}
|
|
|
|
const magnitude = Math.sqrt(norm1) * Math.sqrt(norm2);
|
|
if (magnitude === 0) return 0;
|
|
return dotProduct / magnitude;
|
|
}
|
|
}
|
|
|
|
// Export singleton instance
|
|
export default new ContextService();
|