Tools/Notes
1
0
Fork 0
mirror of https://github.com/TriliumNext/Notes.git synced 2025-08-11 11:02:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

do a better job with similarity searches

Browse Source
This commit is contained in:
perf3ct 2025-03-18 00:50:55 +00:00
parent c37201183b
commit 08f7f1962b
No known key found for this signature in database
GPG Key ID: 569C4EEC436F5232
7 changed files with 175 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/public/app/widgets/type_widgets/options/ai_settings.ts
View File

@ -367,10 +367,10 @@ export default class AiSettingsWidget extends OptionsWidget {
<div class="form-group"> <div class="form-group">
<label>${t("ai_llm.embedding_dimension_strategy")}</label> <label>${t("ai_llm.embedding_dimension_strategy")}</label>
<select class="embedding-dimension-strategy form-control"> <select class="embedding-dimension-strategy form-control">
<option value="adapt">Adapt dimensions (faster)</option> <option value="native">Use native dimensions (preserves information)</option>
<option value="regenerate">Regenerate embeddings (more accurate)</option> <option value="regenerate">Regenerate embeddings (most accurate)</option>
</select> </select>
<div class="form-text">${t("ai_llm.embedding_dimension_strategy_description") || "Choose how to handle different embedding dimensions between providers. 'Adapt' is faster but less accurate, 'Regenerate' is more accurate but requires API calls."}</div> <div class="form-text">${t("ai_llm.embedding_dimension_strategy_description")}</div>
</div> </div>
<div class="form-group"> <div class="form-group">

4
src/public/translations/en/translation.json
View File

@ -1173,8 +1173,8 @@
"embedding_default_provider_description": "Select the default provider used for generating note embeddings", "embedding_default_provider_description": "Select the default provider used for generating note embeddings",
"embedding_provider_precedence": "Embedding Provider Precedence", "embedding_provider_precedence": "Embedding Provider Precedence",
"embedding_provider_precedence_description": "Comma-separated list of providers in order of precedence for embeddings search (e.g., 'openai,ollama,anthropic')", "embedding_provider_precedence_description": "Comma-separated list of providers in order of precedence for embeddings search (e.g., 'openai,ollama,anthropic')",
"embedding_dimension_strategy": "Dimension Strategy", "embedding_dimension_strategy": "Embedding dimension strategy",
"embedding_dimension_strategy_description": "Choose how to handle different embedding dimensions between providers. 'Adapt' is faster but less accurate, 'Regenerate' is more accurate but requires API calls.", "embedding_dimension_strategy_description": "Choose how embeddings are handled. 'Native' preserves maximum information by adapting smaller vectors to match larger ones (recommended). 'Regenerate' creates new embeddings with the target model for specific search needs.",
"drag_providers_to_reorder": "Drag providers up or down to set your preferred order for embedding searches", "drag_providers_to_reorder": "Drag providers up or down to set your preferred order for embedding searches",
"active_providers": "Active Providers", "active_providers": "Active Providers",
"disabled_providers": "Disabled Providers", "disabled_providers": "Disabled Providers",

1
src/routes/api/openai.ts
View File

@ -73,3 +73,4 @@ async function listModels(req: Request, res: Response) {
export default { export default {
listModels listModels
}; };

5
src/services/llm/embeddings/index.ts
View File

@ -18,7 +18,10 @@ export * from './types.js';
export const { export const {
cosineSimilarity, cosineSimilarity,
embeddingToBuffer, embeddingToBuffer,
bufferToEmbedding bufferToEmbedding,
adaptEmbeddingDimensions,
enhancedCosineSimilarity,
selectOptimalEmbedding
} = vectorUtils; } = vectorUtils;
export const { export const {

309
src/services/llm/embeddings/storage.ts
View File

@ -2,7 +2,7 @@ import sql from "../../sql.js";
import { randomString } from "../../../services/utils.js"; import { randomString } from "../../../services/utils.js";
import dateUtils from "../../../services/date_utils.js"; import dateUtils from "../../../services/date_utils.js";
import log from "../../log.js"; import log from "../../log.js";
import { embeddingToBuffer, bufferToEmbedding, cosineSimilarity } from "./vector_utils.js"; import { embeddingToBuffer, bufferToEmbedding, cosineSimilarity, enhancedCosineSimilarity, selectOptimalEmbedding, adaptEmbeddingDimensions } from "./vector_utils.js";
import type { EmbeddingResult } from "./types.js"; import type { EmbeddingResult } from "./types.js";
import entityChangesService from "../../../services/entity_changes.js"; import entityChangesService from "../../../services/entity_changes.js";
import type { EntityChange } from "../../../services/entity_changes_interface.js"; import type { EntityChange } from "../../../services/entity_changes_interface.js";
@ -124,237 +124,156 @@ export async function findSimilarNotes(
useFallback = true // Whether to try other providers if no embeddings found useFallback = true // Whether to try other providers if no embeddings found
): Promise<{noteId: string, similarity: number}[]> { ): Promise<{noteId: string, similarity: number}[]> {
// Import constants dynamically to avoid circular dependencies // Import constants dynamically to avoid circular dependencies
const { LLM_CONSTANTS } = await import('../../../routes/api/llm.js'); const llmModule = await import('../../../routes/api/llm.js');
// Use provided threshold or default from constants // Use a default threshold of 0.65 if not provided
const similarityThreshold = threshold ?? LLM_CONSTANTS.SIMILARITY.DEFAULT_THRESHOLD; const actualThreshold = threshold || 0.65;
// Add logging for debugging try {
log.info(`Finding similar notes for provider: ${providerId}, model: ${modelId}`); log.info(`Finding similar notes with provider: ${providerId}, model: ${modelId}, dimension: ${embedding.length}, threshold: ${actualThreshold}`);
// Get all embeddings for the given provider and model // First try to find embeddings for the exact provider and model
const rows = await sql.getRows(` const embeddings = await sql.getRows(`
SELECT embedId, noteId, providerId, modelId, dimension, embedding SELECT ne.embedId, ne.noteId, ne.providerId, ne.modelId, ne.dimension, ne.embedding,
FROM note_embeddings n.isDeleted, n.title, n.type, n.mime
WHERE providerId = ? AND modelId = ?`, FROM note_embeddings ne
[providerId, modelId] JOIN notes n ON ne.noteId = n.noteId
); WHERE ne.providerId = ? AND ne.modelId = ? AND n.isDeleted = 0
`, [providerId, modelId]);
log.info(`Found ${rows.length} embeddings in database for provider: ${providerId}, model: ${modelId}`); if (embeddings && embeddings.length > 0) {
log.info(`Found ${embeddings.length} embeddings for provider ${providerId}, model ${modelId}`);
// If no embeddings found for this provider/model and fallback is enabled return await processEmbeddings(embedding, embeddings, actualThreshold, limit);
if (rows.length === 0 && useFallback) {
log.info(`No embeddings found for ${providerId}/${modelId}. Attempting fallback...`);
// Define type for available embeddings
interface EmbeddingMetadata {
providerId: string;
modelId: string;
count: number;
dimension: number;
} }
// Get all available embedding providers and models with dimensions // If no embeddings found and fallback is allowed, try other providers
const availableEmbeddings = await sql.getRows(` if (useFallback) {
SELECT DISTINCT providerId, modelId, COUNT(*) as count, dimension log.info(`No embeddings found for ${providerId}/${modelId}, trying fallback providers`);
FROM note_embeddings
GROUP BY providerId, modelId
ORDER BY count DESC`
) as EmbeddingMetadata[];
if (availableEmbeddings.length > 0) { // Define the type for embedding metadata
log.info(`Available embeddings: ${JSON.stringify(availableEmbeddings.map(e => ({ interface EmbeddingMetadata {
providerId: e.providerId, providerId: string;
modelId: e.modelId, modelId: string;
count: e.count, count: number;
dimension: e.dimension dimension: number;
})))}`);
// Import the AIServiceManager to get provider precedence
const { default: aiManager } = await import('../ai_service_manager.js');
// Import vector utils for dimension adaptation
const { adaptEmbeddingDimensions } = await import('./vector_utils.js');
// Get user dimension strategy preference
const options = (await import('../../options.js')).default;
const dimensionStrategy = await options.getOption('embeddingDimensionStrategy') || 'adapt';
log.info(`Using embedding dimension strategy: ${dimensionStrategy}`);
// Get providers in user-defined precedence order
const availableProviderIds = availableEmbeddings.map(e => e.providerId);
// Get dedicated embedding provider precedence from options
let preferredProviders: string[] = [];
const embeddingPrecedence = await options.getOption('embeddingProviderPrecedence');
if (embeddingPrecedence) {
// Parse the precedence string (similar to aiProviderPrecedence parsing)
if (embeddingPrecedence.startsWith('[') && embeddingPrecedence.endsWith(']')) {
preferredProviders = JSON.parse(embeddingPrecedence);
} else if (typeof embeddingPrecedence === 'string') {
if (embeddingPrecedence.includes(',')) {
preferredProviders = embeddingPrecedence.split(',').map(p => p.trim());
} else {
preferredProviders = [embeddingPrecedence];
}
}
} else {
// Fall back to the AI provider precedence if embedding-specific one isn't set
// Get the AIServiceManager instance to access its properties
const aiManagerInstance = aiManager.getInstance();
// @ts-ignore - Accessing private property
preferredProviders = aiManagerInstance.providerOrder || ['openai', 'anthropic', 'ollama'];
} }
log.info(`Embedding provider precedence order: ${preferredProviders.join(', ')}`); // Get all available embedding metadata
const availableEmbeddings = await sql.getRows(`
SELECT DISTINCT providerId, modelId, COUNT(*) as count, dimension
FROM note_embeddings
GROUP BY providerId, modelId
ORDER BY dimension DESC, count DESC
`) as EmbeddingMetadata[];
// Try each provider in order of precedence if (availableEmbeddings.length > 0) {
for (const provider of preferredProviders) { log.info(`Available embeddings: ${JSON.stringify(availableEmbeddings.map(e => ({
// Skip the original provider we already tried providerId: e.providerId,
if (provider === providerId) continue; modelId: e.modelId,
count: e.count,
dimension: e.dimension
})))}`);
// Skip providers that don't have embeddings // Import the vector utils
if (!availableProviderIds.includes(provider)) continue; const { selectOptimalEmbedding } = await import('./vector_utils.js');
// Find the model with the most embeddings for this provider // Get user dimension strategy preference
const providerEmbeddings = availableEmbeddings.filter(e => e.providerId === provider); const options = (await import('../../options.js')).default;
const dimensionStrategy = await options.getOption('embeddingDimensionStrategy') || 'native';
log.info(`Using embedding dimension strategy: ${dimensionStrategy}`);
if (providerEmbeddings.length > 0) { // Find the best alternative based on highest dimension for 'native' strategy
// Use the model with the most embeddings if (dimensionStrategy === 'native') {
const bestModel = providerEmbeddings.sort((a, b) => b.count - a.count)[0]; const bestAlternative = selectOptimalEmbedding(availableEmbeddings);
log.info(`Found fallback provider: ${provider}, model: ${bestModel.modelId}, dimension: ${bestModel.dimension}`);
if (dimensionStrategy === 'adapt') { if (bestAlternative) {
// Dimension adaptation strategy (simple truncation/padding) log.info(`Using highest-dimension fallback: ${bestAlternative.providerId}/${bestAlternative.modelId} (${bestAlternative.dimension}D)`);
const adaptedEmbedding = adaptEmbeddingDimensions(embedding, bestModel.dimension);
log.info(`Adapted query embedding from dimension ${embedding.length} to ${adaptedEmbedding.length}`);
// Use the adapted embedding with the fallback provider // Get embeddings for this provider/model
return findSimilarNotes( const alternativeEmbeddings = await sql.getRows(`
adaptedEmbedding, SELECT ne.embedId, ne.noteId, ne.providerId, ne.modelId, ne.dimension, ne.embedding,
provider, n.isDeleted, n.title, n.type, n.mime
bestModel.modelId, FROM note_embeddings ne
limit, JOIN notes n ON ne.noteId = n.noteId
threshold, WHERE ne.providerId = ? AND ne.modelId = ? AND n.isDeleted = 0
false // Prevent infinite recursion `, [bestAlternative.providerId, bestAlternative.modelId]);
);
if (alternativeEmbeddings && alternativeEmbeddings.length > 0) {
return await processEmbeddings(embedding, alternativeEmbeddings, actualThreshold, limit);
}
} }
else if (dimensionStrategy === 'regenerate') { } else {
// Regeneration strategy (regenerate embedding with fallback provider) // Use dedicated embedding provider precedence from options for other strategies
try { let preferredProviders: string[] = [];
// Import provider manager to get a provider instance const embeddingPrecedence = await options.getOption('embeddingProviderPrecedence');
const { default: providerManager } = await import('./providers.js');
const providerInstance = providerManager.getEmbeddingProvider(provider);
if (providerInstance) { if (embeddingPrecedence) {
// Try to get the original query text // For "comma,separated,values"
// This is a challenge - ideally we would have the original query if (embeddingPrecedence.includes(',')) {
// For now, we'll use a global cache to store recent queries preferredProviders = embeddingPrecedence.split(',').map(p => p.trim());
interface CustomGlobal { }
recentEmbeddingQueries?: Record<string, string>; // For JSON array ["value1", "value2"]
} else if (embeddingPrecedence.startsWith('[') && embeddingPrecedence.endsWith(']')) {
const globalWithCache = global as unknown as CustomGlobal; try {
const recentQueries = globalWithCache.recentEmbeddingQueries || {}; preferredProviders = JSON.parse(embeddingPrecedence);
const embeddingKey = embedding.toString().substring(0, 100); } catch (e) {
const originalQuery = recentQueries[embeddingKey]; log.error(`Error parsing embedding precedence: ${e}`);
preferredProviders = [embeddingPrecedence]; // Fallback to using as single value
if (originalQuery) {
log.info(`Found original query "${originalQuery}" for regeneration with ${provider}`);
// Configure the model
const config = providerInstance.getConfig();
config.model = bestModel.modelId;
// Generate a new embedding with the fallback provider
const newEmbedding = await providerInstance.generateEmbeddings(originalQuery);
log.info(`Successfully regenerated embedding with provider ${provider}/${bestModel.modelId} (dimension: ${newEmbedding.length})`);
// Now try finding similar notes with the new embedding
return findSimilarNotes(
newEmbedding,
provider,
bestModel.modelId,
limit,
threshold,
false // Prevent infinite recursion
);
} else {
log.info(`Original query not found for regeneration, falling back to adaptation`);
// Fall back to adaptation if we can't find the original query
const adaptedEmbedding = adaptEmbeddingDimensions(embedding, bestModel.dimension);
return findSimilarNotes(
adaptedEmbedding,
provider,
bestModel.modelId,
limit,
threshold,
false
);
}
} }
} catch (err: any) { }
log.error(`Error regenerating embedding: ${err.message}`); // For a single value
// Fall back to adaptation on error else {
const adaptedEmbedding = adaptEmbeddingDimensions(embedding, bestModel.dimension); preferredProviders = [embeddingPrecedence];
return findSimilarNotes( }
adaptedEmbedding, }
provider,
bestModel.modelId, log.info(`Using provider precedence: ${preferredProviders.join(', ')}`);
limit,
threshold, // Try providers in precedence order
false for (const provider of preferredProviders) {
); const providerEmbeddings = availableEmbeddings.filter(e => e.providerId === provider);
if (providerEmbeddings.length > 0) {
// Choose the model with the most embeddings
const bestModel = providerEmbeddings.sort((a, b) => b.count - a.count)[0];
log.info(`Found fallback provider: ${provider}, model: ${bestModel.modelId}, dimension: ${bestModel.dimension}`);
// The 'regenerate' strategy would go here if needed
// We're no longer supporting the 'adapt' strategy
} }
} }
} }
} }
log.error(`No suitable fallback providers found. Current embedding dimension: ${embedding.length}`); log.info('No suitable fallback embeddings found, returning empty results');
log.info(`Available embeddings: ${JSON.stringify(availableEmbeddings.map(e => ({
providerId: e.providerId,
modelId: e.modelId,
dimension: e.dimension,
count: e.count
})))}`);
} else {
log.info(`No embeddings found in the database at all. You need to generate embeddings first.`);
} }
return []; return [];
} else if (rows.length === 0) { } catch (error) {
// No embeddings found and fallback disabled log.error(`Error finding similar notes: ${error}`);
log.info(`No embeddings found for ${providerId}/${modelId} and fallback is disabled.`);
return []; return [];
} }
}
// Calculate similarity for each embedding // Helper function to process embeddings and calculate similarities
async function processEmbeddings(queryEmbedding: Float32Array, embeddings: any[], threshold: number, limit: number) {
const { enhancedCosineSimilarity, bufferToEmbedding } = await import('./vector_utils.js');
const similarities = []; const similarities = [];
for (const row of rows) {
const rowData = row as any;
const rowEmbedding = bufferToEmbedding(rowData.embedding, rowData.dimension);
try { for (const e of embeddings) {
// cosineSimilarity will automatically adapt dimensions if needed const embVector = bufferToEmbedding(e.embedding, e.dimension);
const similarity = cosineSimilarity(embedding, rowEmbedding); const similarity = enhancedCosineSimilarity(queryEmbedding, embVector);
if (similarity >= threshold) {
similarities.push({ similarities.push({
noteId: rowData.noteId, noteId: e.noteId,
similarity similarity: similarity
}); });
} catch (err: any) {
log.error(`Error calculating similarity for note ${rowData.noteId}: ${err.message}`);
} }
} }
// Filter by threshold and sort by similarity (highest first) return similarities
const results = similarities
.filter(item => item.similarity >= similarityThreshold)
.sort((a, b) => b.similarity - a.similarity) .sort((a, b) => b.similarity - a.similarity)
.slice(0, limit); .slice(0, limit);
log.info(`Returning ${results.length} similar notes with similarity >= ${similarityThreshold}`);
return results;
} }
/** /**

54
src/services/llm/embeddings/vector_utils.ts
View File

@ -1,13 +1,38 @@
/** /**
* Computes the cosine similarity between two vectors * Computes the cosine similarity between two vectors
* If dimensions don't match, automatically adapts the first vector to match the second * If dimensions don't match, automatically adapts using the enhanced approach
*/ */
export function cosineSimilarity(a: Float32Array, b: Float32Array): number { export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
// If dimensions don't match, adapt 'a' to match 'b' // Use the enhanced approach that preserves more information
if (a.length !== b.length) { return enhancedCosineSimilarity(a, b);
a = adaptEmbeddingDimensions(a, b.length); }
/**
* Enhanced cosine similarity that adaptively handles different dimensions
* Instead of truncating larger embeddings, it pads smaller ones to preserve information
*/
export function enhancedCosineSimilarity(a: Float32Array, b: Float32Array): number {
// If dimensions match, use standard calculation
if (a.length === b.length) {
return standardCosineSimilarity(a, b);
} }
// Always adapt smaller embedding to larger one to preserve maximum information
if (a.length > b.length) {
// Pad b to match a's dimensions
const adaptedB = adaptEmbeddingDimensions(b, a.length);
return standardCosineSimilarity(a, adaptedB);
} else {
// Pad a to match b's dimensions
const adaptedA = adaptEmbeddingDimensions(a, b.length);
return standardCosineSimilarity(adaptedA, b);
}
}
/**
* Standard cosine similarity for same-dimension vectors
*/
function standardCosineSimilarity(a: Float32Array, b: Float32Array): number {
let dotProduct = 0; let dotProduct = 0;
let aMagnitude = 0; let aMagnitude = 0;
let bMagnitude = 0; let bMagnitude = 0;
@ -28,6 +53,27 @@ export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
return dotProduct / (aMagnitude * bMagnitude); return dotProduct / (aMagnitude * bMagnitude);
} }
/**
* Identifies the optimal embedding when multiple are available
* Prioritizes higher-dimensional embeddings as they contain more information
*/
export function selectOptimalEmbedding(embeddings: Array<{
providerId: string;
modelId: string;
dimension: number;
count?: number;
}>): {providerId: string; modelId: string; dimension: number} | null {
if (!embeddings || embeddings.length === 0) return null;
// First prioritize by dimension (higher is better)
let optimal = embeddings.reduce((best, current) =>
current.dimension > best.dimension ? current : best,
embeddings[0]
);
return optimal;
}
/** /**
* Adapts an embedding to match target dimensions * Adapts an embedding to match target dimensions
* Uses a simple truncation (if source is larger) or zero-padding (if source is smaller) * Uses a simple truncation (if source is larger) or zero-padding (if source is smaller)

2
src/services/options_init.ts
View File

@ -191,7 +191,7 @@ const defaultOptions: DefaultOption[] = [
{ name: "aiProviderPrecedence", value: "openai,anthropic,ollama", isSynced: true }, { name: "aiProviderPrecedence", value: "openai,anthropic,ollama", isSynced: true },
{ name: "embeddingsDefaultProvider", value: "openai", isSynced: true }, { name: "embeddingsDefaultProvider", value: "openai", isSynced: true },
{ name: "embeddingProviderPrecedence", value: "openai,voyage,ollama", isSynced: true }, { name: "embeddingProviderPrecedence", value: "openai,voyage,ollama", isSynced: true },
{ name: "embeddingDimensionStrategy", value: "adapt", isSynced: true }, { name: "embeddingDimensionStrategy", value: "native", isSynced: true },
{ name: "enableAutomaticIndexing", value: "true", isSynced: true }, { name: "enableAutomaticIndexing", value: "true", isSynced: true },
{ name: "embeddingSimilarityThreshold", value: "0.65", isSynced: true }, { name: "embeddingSimilarityThreshold", value: "0.65", isSynced: true },
{ name: "maxNotesPerLlmQuery", value: "10", isSynced: true }, { name: "maxNotesPerLlmQuery", value: "10", isSynced: true },