I can create embeddings now?

This commit is contained in:
perf3ct 2025-03-08 23:13:49 +00:00
parent 6ace4d5692
commit 0daa9e717f
No known key found for this signature in database
GPG Key ID: 569C4EEC436F5232
2 changed files with 111 additions and 159 deletions

View File

@ -17,33 +17,32 @@ async function findSimilarNotes(req: Request, res: Response) {
const threshold = parseFloat(req.query.threshold as string || '0.7');
if (!noteId) {
return res.status(400).send({
return [400, {
success: false,
message: "Note ID is required"
});
}];
}
try {
const embedding = await vectorStore.getEmbeddingForNote(noteId, providerId, modelId);
if (!embedding) {
// If no embedding exists for this note yet, generate one
const note = becca.getNote(noteId);
if (!note) {
return res.status(404).send({
return [404, {
success: false,
message: "Note not found"
});
}];
}
const context = await vectorStore.getNoteEmbeddingContext(noteId);
const provider = providerManager.getEmbeddingProvider(providerId);
if (!provider) {
return res.status(400).send({
return [400, {
success: false,
message: `Embedding provider '${providerId}' not found`
});
}];
}
const newEmbedding = await provider.generateNoteEmbeddings(context);
@ -53,26 +52,20 @@ async function findSimilarNotes(req: Request, res: Response) {
newEmbedding, providerId, modelId, limit, threshold
);
return res.send({
return {
success: true,
similarNotes
});
};
}
const similarNotes = await vectorStore.findSimilarNotes(
embedding.embedding, providerId, modelId, limit, threshold
);
return res.send({
return {
success: true,
similarNotes
});
} catch (error: any) {
return res.status(500).send({
success: false,
message: error.message || "Unknown error"
});
}
};
}
/**
@ -86,20 +79,19 @@ async function searchByText(req: Request, res: Response) {
const threshold = parseFloat(req.query.threshold as string || '0.7');
if (!text) {
return res.status(400).send({
return [400, {
success: false,
message: "Search text is required"
});
}];
}
try {
const provider = providerManager.getEmbeddingProvider(providerId);
if (!provider) {
return res.status(400).send({
return [400, {
success: false,
message: `Embedding provider '${providerId}' not found`
});
}];
}
// Generate embedding for the search text
@ -110,34 +102,22 @@ async function searchByText(req: Request, res: Response) {
embedding, providerId, modelId, limit, threshold
);
return res.send({
return {
success: true,
similarNotes
});
} catch (error: any) {
return res.status(500).send({
success: false,
message: error.message || "Unknown error"
});
}
};
}
/**
* Get embedding providers
*/
async function getProviders(req: Request, res: Response) {
try {
const providerConfigs = await providerManager.getEmbeddingProviderConfigs();
return res.send({
return {
success: true,
providers: providerConfigs
});
} catch (error: any) {
return res.status(500).send({
success: false,
message: error.message || "Unknown error"
});
}
};
}
/**
@ -147,43 +127,27 @@ async function updateProvider(req: Request, res: Response) {
const { providerId } = req.params;
const { isEnabled, priority, config } = req.body;
try {
const success = await providerManager.updateEmbeddingProviderConfig(
providerId, isEnabled, priority, config
);
if (!success) {
return res.status(404).send({
return [404, {
success: false,
message: "Provider not found"
});
}];
}
return res.send({
return {
success: true
});
} catch (error: any) {
return res.status(500).send({
success: false,
message: error.message || "Unknown error"
});
}
};
}
/**
* Manually trigger a reprocessing of all notes
*/
async function reprocessAllNotes(req: Request, res: Response) {
// Wrap in a try-catch to handle errors
try {
// Start the reprocessing operation in the background
// and immediately respond to the client
res.send({
success: true,
message: "Embedding reprocessing started in the background"
});
// Continue processing in the background after sending the response
setTimeout(async () => {
try {
await vectorStore.reprocessAllNotes();
@ -192,24 +156,18 @@ async function reprocessAllNotes(req: Request, res: Response) {
log.error(`Error during background embedding reprocessing: ${error.message || "Unknown error"}`);
}
}, 0);
} catch (error: any) {
// Only catch errors that happen before we send the response
log.error(`Error initiating embedding reprocessing: ${error.message || "Unknown error"}`);
if (!res.headersSent) {
res.status(500).send({
success: false,
message: error.message || "Unknown error"
});
}
}
// Return the response data
return {
success: true,
message: "Embedding reprocessing started in the background"
};
}
/**
* Get embedding queue status
*/
async function getQueueStatus(req: Request, res: Response) {
try {
// Use the imported sql instead of requiring it
const queueCount = await sql.getValue(
"SELECT COUNT(*) FROM embedding_queue"
@ -223,20 +181,14 @@ async function getQueueStatus(req: Request, res: Response) {
"SELECT COUNT(*) FROM note_embeddings"
);
return res.send({
return {
success: true,
status: {
queueCount,
failedCount,
totalEmbeddingsCount
}
});
} catch (error: any) {
return res.status(500).send({
success: false,
message: error.message || "Unknown error"
});
}
};
}
export default {

View File

@ -372,12 +372,12 @@ function register(app: express.Application) {
etapiBackupRoute.register(router);
// Embeddings API endpoints
route(GET, "/api/embeddings/similar/:noteId", [auth.checkApiAuth], embeddingsRoute.findSimilarNotes, apiResultHandler);
route(PST, "/api/embeddings/search", [auth.checkApiAuth, csrfMiddleware], embeddingsRoute.searchByText, apiResultHandler);
route(GET, "/api/embeddings/providers", [auth.checkApiAuth], embeddingsRoute.getProviders, apiResultHandler);
route(PATCH, "/api/embeddings/providers/:providerId", [auth.checkApiAuth, csrfMiddleware], embeddingsRoute.updateProvider, apiResultHandler);
route(PST, "/api/embeddings/reprocess", [auth.checkApiAuth, csrfMiddleware], embeddingsRoute.reprocessAllNotes, apiResultHandler);
route(GET, "/api/embeddings/queue-status", [auth.checkApiAuth], embeddingsRoute.getQueueStatus, apiResultHandler);
apiRoute(GET, "/api/embeddings/similar/:noteId", embeddingsRoute.findSimilarNotes);
apiRoute(PST, "/api/embeddings/search", embeddingsRoute.searchByText);
apiRoute(GET, "/api/embeddings/providers", embeddingsRoute.getProviders);
apiRoute(PATCH, "/api/embeddings/providers/:providerId", embeddingsRoute.updateProvider);
apiRoute(PST, "/api/embeddings/reprocess", embeddingsRoute.reprocessAllNotes);
apiRoute(GET, "/api/embeddings/queue-status", embeddingsRoute.getQueueStatus);
// Ollama API endpoints
route(PST, "/api/ollama/list-models", [auth.checkApiAuth, csrfMiddleware], ollamaRoute.listModels, apiResultHandler);