mirror of
https://github.com/TriliumNext/Notes.git
synced 2025-07-31 12:12:28 +08:00
"rebuild index" functionality for users
This commit is contained in:
parent
72b1426d94
commit
eaa947ef7c
@ -47,6 +47,7 @@ interface FailedEmbeddingNotes {
|
||||
|
||||
export default class AiSettingsWidget extends OptionsWidget {
|
||||
private statsRefreshInterval: NodeJS.Timeout | null = null;
|
||||
private indexRebuildRefreshInterval: NodeJS.Timeout | null = null;
|
||||
private readonly STATS_REFRESH_INTERVAL = 5000; // 5 seconds
|
||||
|
||||
doRender() {
|
||||
@ -243,6 +244,17 @@ export default class AiSettingsWidget extends OptionsWidget {
|
||||
${t("ai_llm.reprocess_index")}
|
||||
</button>
|
||||
<div class="help-text">${t("ai_llm.reprocess_index_description")}</div>
|
||||
|
||||
<!-- Index rebuild progress tracking -->
|
||||
<div class="index-rebuild-progress-container mt-2" style="display: none;">
|
||||
<div class="mt-2">
|
||||
<strong>${t("ai_llm.index_rebuild_progress")}:</strong> <span class="index-rebuild-status-text">-</span>
|
||||
</div>
|
||||
<div class="progress mt-1" style="height: 10px;">
|
||||
<div class="progress-bar index-rebuild-progress" role="progressbar" style="width: 0%;"
|
||||
aria-valuenow="0" aria-valuemin="0" aria-valuemax="100">0%</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
@ -476,7 +488,10 @@ export default class AiSettingsWidget extends OptionsWidget {
|
||||
try {
|
||||
await server.post('embeddings/rebuild-index');
|
||||
toastService.showMessage(t("ai_llm.reprocess_index_started"));
|
||||
// Refresh stats after reprocessing starts
|
||||
// Start tracking index rebuild progress
|
||||
await this.refreshIndexRebuildStatus();
|
||||
|
||||
// Also refresh embedding stats since they'll update as embeddings are processed
|
||||
await this.refreshEmbeddingStats();
|
||||
} catch (error) {
|
||||
console.error("Error rebuilding index:", error);
|
||||
@ -567,6 +582,9 @@ export default class AiSettingsWidget extends OptionsWidget {
|
||||
this.$widget.find('.embedding-section').is(':visible')) {
|
||||
await this.refreshEmbeddingStats(true);
|
||||
|
||||
// Also check index rebuild status
|
||||
await this.refreshIndexRebuildStatus(true);
|
||||
|
||||
// Also update failed embeddings list periodically
|
||||
await this.updateFailedEmbeddingsList();
|
||||
}
|
||||
@ -581,6 +599,11 @@ export default class AiSettingsWidget extends OptionsWidget {
|
||||
clearInterval(this.statsRefreshInterval);
|
||||
this.statsRefreshInterval = null;
|
||||
}
|
||||
|
||||
if (this.indexRebuildRefreshInterval) {
|
||||
clearInterval(this.indexRebuildRefreshInterval);
|
||||
this.indexRebuildRefreshInterval = null;
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up when the widget is removed
|
||||
@ -699,6 +722,11 @@ export default class AiSettingsWidget extends OptionsWidget {
|
||||
if (stats.failedNotesCount > 0 && !silent) {
|
||||
await this.updateFailedEmbeddingsList();
|
||||
}
|
||||
|
||||
// Also check index rebuild status if not in silent mode
|
||||
if (!silent) {
|
||||
await this.refreshIndexRebuildStatus(silent);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error fetching embedding stats:", error);
|
||||
@ -715,6 +743,83 @@ export default class AiSettingsWidget extends OptionsWidget {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Refresh the index rebuild status
|
||||
*/
|
||||
async refreshIndexRebuildStatus(silent = false) {
|
||||
if (!this.$widget) return;
|
||||
|
||||
try {
|
||||
// Get the current status of index rebuilding
|
||||
const response = await server.get('embeddings/index-rebuild-status') as {
|
||||
success: boolean,
|
||||
status: {
|
||||
inProgress: boolean,
|
||||
progress: number,
|
||||
total: number,
|
||||
current: number
|
||||
}
|
||||
};
|
||||
|
||||
if (response && response.success) {
|
||||
const status = response.status;
|
||||
const $progressContainer = this.$widget.find('.index-rebuild-progress-container');
|
||||
const $progressBar = this.$widget.find('.index-rebuild-progress');
|
||||
const $statusText = this.$widget.find('.index-rebuild-status-text');
|
||||
|
||||
// Only show the progress container if rebuild is in progress
|
||||
if (status.inProgress) {
|
||||
$progressContainer.show();
|
||||
} else if (status.progress === 100) {
|
||||
// Show for 10 seconds after completion, then hide
|
||||
$progressContainer.show();
|
||||
setTimeout(() => {
|
||||
$progressContainer.fadeOut('slow');
|
||||
}, 10000);
|
||||
} else if (status.progress === 0) {
|
||||
// Hide if no rebuild has been started
|
||||
$progressContainer.hide();
|
||||
}
|
||||
|
||||
// Update progress bar
|
||||
$progressBar.css('width', `${status.progress}%`);
|
||||
$progressBar.attr('aria-valuenow', status.progress.toString());
|
||||
$progressBar.text(`${status.progress}%`);
|
||||
|
||||
// Update status text
|
||||
if (status.inProgress) {
|
||||
$statusText.text(t("ai_llm.index_rebuilding", { percentage: status.progress }));
|
||||
|
||||
// Apply animated style for active progress
|
||||
$progressBar.addClass('progress-bar-striped progress-bar-animated bg-info');
|
||||
$progressBar.removeClass('bg-success');
|
||||
} else if (status.progress === 100) {
|
||||
$statusText.text(t("ai_llm.index_rebuild_complete"));
|
||||
|
||||
// Apply success style for completed progress
|
||||
$progressBar.removeClass('progress-bar-striped progress-bar-animated bg-info');
|
||||
$progressBar.addClass('bg-success');
|
||||
}
|
||||
|
||||
// Start a refresh interval if in progress
|
||||
if (status.inProgress && !this.indexRebuildRefreshInterval) {
|
||||
this.indexRebuildRefreshInterval = setInterval(() => {
|
||||
this.refreshIndexRebuildStatus(true);
|
||||
}, this.STATS_REFRESH_INTERVAL);
|
||||
} else if (!status.inProgress && this.indexRebuildRefreshInterval) {
|
||||
// Clear the interval if rebuild is complete
|
||||
clearInterval(this.indexRebuildRefreshInterval);
|
||||
this.indexRebuildRefreshInterval = null;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error fetching index rebuild status:", error);
|
||||
if (!silent) {
|
||||
toastService.showError(t("ai_llm.index_rebuild_status_error"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async updateFailedEmbeddingsList() {
|
||||
if (!this.$widget) return;
|
||||
|
||||
|
@ -1186,6 +1186,11 @@
|
||||
"reprocess_index_started": "Index rebuilding started in the background",
|
||||
"reprocess_index_error": "Error rebuilding search index",
|
||||
|
||||
"index_rebuild_progress": "Index Rebuild Progress",
|
||||
"index_rebuilding": "Rebuilding index ({{percentage}}%)",
|
||||
"index_rebuild_complete": "Index rebuild complete",
|
||||
"index_rebuild_status_error": "Error checking index rebuild status",
|
||||
|
||||
"embedding_statistics": "Embedding Statistics",
|
||||
"total_notes": "Total Notes",
|
||||
"processed_notes": "Processed Notes",
|
||||
|
@ -279,6 +279,18 @@ async function rebuildIndex(req: Request, res: Response) {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current index rebuild status
|
||||
*/
|
||||
async function getIndexRebuildStatus(req: Request, res: Response) {
|
||||
const status = indexService.getIndexRebuildStatus();
|
||||
|
||||
return {
|
||||
success: true,
|
||||
status
|
||||
};
|
||||
}
|
||||
|
||||
export default {
|
||||
findSimilarNotes,
|
||||
searchByText,
|
||||
@ -290,5 +302,6 @@ export default {
|
||||
getFailedNotes,
|
||||
retryFailedNote,
|
||||
retryAllFailedNotes,
|
||||
rebuildIndex
|
||||
rebuildIndex,
|
||||
getIndexRebuildStatus
|
||||
};
|
||||
|
@ -384,6 +384,7 @@ function register(app: express.Application) {
|
||||
apiRoute(PST, "/api/embeddings/retry/:noteId", embeddingsRoute.retryFailedNote);
|
||||
apiRoute(PST, "/api/embeddings/retry-all-failed", embeddingsRoute.retryAllFailedNotes);
|
||||
apiRoute(PST, "/api/embeddings/rebuild-index", embeddingsRoute.rebuildIndex);
|
||||
apiRoute(GET, "/api/embeddings/index-rebuild-status", embeddingsRoute.getIndexRebuildStatus);
|
||||
|
||||
// LLM chat session management endpoints
|
||||
apiRoute(PST, "/api/llm/sessions", llmRoute.createSession);
|
||||
|
@ -8,6 +8,7 @@ import { getNoteEmbeddingContext } from "./content_processing.js";
|
||||
import { deleteNoteEmbeddings } from "./storage.js";
|
||||
import type { QueueItem } from "./types.js";
|
||||
import { getChunkingOperations } from "./chunking_interface.js";
|
||||
import indexService from '../index_service.js';
|
||||
|
||||
/**
|
||||
* Queues a note for embedding update
|
||||
@ -176,6 +177,9 @@ export async function processEmbeddingQueue() {
|
||||
return;
|
||||
}
|
||||
|
||||
// Track successfully processed notes count for progress reporting
|
||||
let processedCount = 0;
|
||||
|
||||
for (const note of notes) {
|
||||
try {
|
||||
const noteData = note as unknown as QueueItem;
|
||||
@ -248,6 +252,8 @@ export async function processEmbeddingQueue() {
|
||||
"DELETE FROM embedding_queue WHERE noteId = ?",
|
||||
[noteData.noteId]
|
||||
);
|
||||
// Count as successfully processed
|
||||
processedCount++;
|
||||
} else {
|
||||
// If all providers failed, mark as failed but keep in queue
|
||||
await sql.execute(`
|
||||
@ -286,4 +292,9 @@ export async function processEmbeddingQueue() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update the index rebuild progress if any notes were processed
|
||||
if (processedCount > 0) {
|
||||
indexService.updateIndexRebuildProgress(processedCount);
|
||||
}
|
||||
}
|
||||
|
@ -18,6 +18,7 @@ import { ContextExtractor } from "./context/index.js";
|
||||
import eventService from "../events.js";
|
||||
import type { NoteEmbeddingContext } from "./embeddings/embeddings_interface.js";
|
||||
import type { OptionDefinitions } from "../options_interface.js";
|
||||
import sql from "../sql.js";
|
||||
|
||||
class IndexService {
|
||||
private initialized = false;
|
||||
@ -25,6 +26,12 @@ class IndexService {
|
||||
private contextExtractor = new ContextExtractor();
|
||||
private automaticIndexingInterval?: NodeJS.Timeout;
|
||||
|
||||
// Index rebuilding tracking
|
||||
private indexRebuildInProgress = false;
|
||||
private indexRebuildProgress = 0;
|
||||
private indexRebuildTotal = 0;
|
||||
private indexRebuildCurrent = 0;
|
||||
|
||||
// Configuration
|
||||
private defaultQueryDepth = 2;
|
||||
private maxNotesPerQuery = 10;
|
||||
@ -195,6 +202,13 @@ class IndexService {
|
||||
|
||||
try {
|
||||
this.indexingInProgress = true;
|
||||
this.indexRebuildInProgress = true;
|
||||
this.indexRebuildProgress = 0;
|
||||
this.indexRebuildCurrent = 0;
|
||||
|
||||
// Reset index rebuild progress
|
||||
const totalNotes = await sql.getValue("SELECT COUNT(*) FROM notes WHERE isDeleted = 0") as number;
|
||||
this.indexRebuildTotal = totalNotes;
|
||||
|
||||
if (force) {
|
||||
// Force reindexing of all notes
|
||||
@ -210,18 +224,55 @@ class IndexService {
|
||||
log.info("Full indexing initiated");
|
||||
} else {
|
||||
log.info(`Skipping full indexing, already at ${stats.percentComplete}% completion`);
|
||||
this.indexRebuildInProgress = false;
|
||||
this.indexRebuildProgress = 100;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
} catch (error: any) {
|
||||
log.error(`Error starting full indexing: ${error.message || "Unknown error"}`);
|
||||
this.indexRebuildInProgress = false;
|
||||
return false;
|
||||
} finally {
|
||||
this.indexingInProgress = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update index rebuild progress
|
||||
* @param processed - Number of notes processed
|
||||
*/
|
||||
updateIndexRebuildProgress(processed: number) {
|
||||
if (!this.indexRebuildInProgress) return;
|
||||
|
||||
this.indexRebuildCurrent += processed;
|
||||
|
||||
if (this.indexRebuildTotal > 0) {
|
||||
this.indexRebuildProgress = Math.min(
|
||||
Math.round((this.indexRebuildCurrent / this.indexRebuildTotal) * 100),
|
||||
100
|
||||
);
|
||||
}
|
||||
|
||||
if (this.indexRebuildCurrent >= this.indexRebuildTotal) {
|
||||
this.indexRebuildInProgress = false;
|
||||
this.indexRebuildProgress = 100;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current index rebuild progress
|
||||
*/
|
||||
getIndexRebuildStatus() {
|
||||
return {
|
||||
inProgress: this.indexRebuildInProgress,
|
||||
progress: this.indexRebuildProgress,
|
||||
total: this.indexRebuildTotal,
|
||||
current: this.indexRebuildCurrent
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a batch indexing job for a limited number of notes
|
||||
* @param batchSize - Maximum number of notes to process
|
||||
|
Loading…
x
Reference in New Issue
Block a user