"rebuild index" functionality for users

src/public/app/widgets/type_widgets/options/ai_settings.ts

@@ -47,6 +47,7 @@ interface FailedEmbeddingNotes {
 
 export default class AiSettingsWidget extends OptionsWidget {
     private statsRefreshInterval: NodeJS.Timeout | null = null;
+    private indexRebuildRefreshInterval: NodeJS.Timeout | null = null;
     private readonly STATS_REFRESH_INTERVAL = 5000; // 5 seconds
 
     doRender() {
@@ -243,6 +244,17 @@ export default class AiSettingsWidget extends OptionsWidget {
                         ${t("ai_llm.reprocess_index")}
                     </button>
                     <div class="help-text">${t("ai_llm.reprocess_index_description")}</div>
+
+                    <!-- Index rebuild progress tracking -->
+                    <div class="index-rebuild-progress-container mt-2" style="display: none;">
+                        <div class="mt-2">
+                            <strong>${t("ai_llm.index_rebuild_progress")}:</strong> <span class="index-rebuild-status-text">-</span>
+                        </div>
+                        <div class="progress mt-1" style="height: 10px;">
+                            <div class="progress-bar index-rebuild-progress" role="progressbar" style="width: 0%;"
+                                aria-valuenow="0" aria-valuemin="0" aria-valuemax="100">0%</div>
+                        </div>
+                    </div>
                 </div>
 
                 <div class="form-group">
@@ -476,7 +488,10 @@ export default class AiSettingsWidget extends OptionsWidget {
             try {
                 await server.post('embeddings/rebuild-index');
                 toastService.showMessage(t("ai_llm.reprocess_index_started"));
-                // Refresh stats after reprocessing starts
+                // Start tracking index rebuild progress
+                await this.refreshIndexRebuildStatus();
+
+                // Also refresh embedding stats since they'll update as embeddings are processed
                 await this.refreshEmbeddingStats();
             } catch (error) {
                 console.error("Error rebuilding index:", error);
@@ -567,6 +582,9 @@ export default class AiSettingsWidget extends OptionsWidget {
                 this.$widget.find('.embedding-section').is(':visible')) {
                 await this.refreshEmbeddingStats(true);
 
+                // Also check index rebuild status
+                await this.refreshIndexRebuildStatus(true);
+
                 // Also update failed embeddings list periodically
                 await this.updateFailedEmbeddingsList();
             }
@@ -581,6 +599,11 @@ export default class AiSettingsWidget extends OptionsWidget {
             clearInterval(this.statsRefreshInterval);
             this.statsRefreshInterval = null;
         }
+
+        if (this.indexRebuildRefreshInterval) {
+            clearInterval(this.indexRebuildRefreshInterval);
+            this.indexRebuildRefreshInterval = null;
+        }
     }
 
     // Clean up when the widget is removed
@@ -699,6 +722,11 @@ export default class AiSettingsWidget extends OptionsWidget {
                 if (stats.failedNotesCount > 0 && !silent) {
                     await this.updateFailedEmbeddingsList();
                 }
+
+                // Also check index rebuild status if not in silent mode
+                if (!silent) {
+                    await this.refreshIndexRebuildStatus(silent);
+                }
             }
         } catch (error) {
             console.error("Error fetching embedding stats:", error);
@@ -715,6 +743,83 @@ export default class AiSettingsWidget extends OptionsWidget {
         }
     }
 
+    /**
+     * Refresh the index rebuild status
+     */
+    async refreshIndexRebuildStatus(silent = false) {
+        if (!this.$widget) return;
+
+        try {
+            // Get the current status of index rebuilding
+            const response = await server.get('embeddings/index-rebuild-status') as {
+                success: boolean,
+                status: {
+                    inProgress: boolean,
+                    progress: number,
+                    total: number,
+                    current: number
+                }
+            };
+
+            if (response && response.success) {
+                const status = response.status;
+                const $progressContainer = this.$widget.find('.index-rebuild-progress-container');
+                const $progressBar = this.$widget.find('.index-rebuild-progress');
+                const $statusText = this.$widget.find('.index-rebuild-status-text');
+
+                // Only show the progress container if rebuild is in progress
+                if (status.inProgress) {
+                    $progressContainer.show();
+                } else if (status.progress === 100) {
+                    // Show for 10 seconds after completion, then hide
+                    $progressContainer.show();
+                    setTimeout(() => {
+                        $progressContainer.fadeOut('slow');
+                    }, 10000);
+                } else if (status.progress === 0) {
+                    // Hide if no rebuild has been started
+                    $progressContainer.hide();
+                }
+
+                // Update progress bar
+                $progressBar.css('width', `${status.progress}%`);
+                $progressBar.attr('aria-valuenow', status.progress.toString());
+                $progressBar.text(`${status.progress}%`);
+
+                // Update status text
+                if (status.inProgress) {
+                    $statusText.text(t("ai_llm.index_rebuilding", { percentage: status.progress }));
+
+                    // Apply animated style for active progress
+                    $progressBar.addClass('progress-bar-striped progress-bar-animated bg-info');
+                    $progressBar.removeClass('bg-success');
+                } else if (status.progress === 100) {
+                    $statusText.text(t("ai_llm.index_rebuild_complete"));
+
+                    // Apply success style for completed progress
+                    $progressBar.removeClass('progress-bar-striped progress-bar-animated bg-info');
+                    $progressBar.addClass('bg-success');
+                }
+
+                // Start a refresh interval if in progress
+                if (status.inProgress && !this.indexRebuildRefreshInterval) {
+                    this.indexRebuildRefreshInterval = setInterval(() => {
+                        this.refreshIndexRebuildStatus(true);
+                    }, this.STATS_REFRESH_INTERVAL);
+                } else if (!status.inProgress && this.indexRebuildRefreshInterval) {
+                    // Clear the interval if rebuild is complete
+                    clearInterval(this.indexRebuildRefreshInterval);
+                    this.indexRebuildRefreshInterval = null;
+                }
+            }
+        } catch (error) {
+            console.error("Error fetching index rebuild status:", error);
+            if (!silent) {
+                toastService.showError(t("ai_llm.index_rebuild_status_error"));
+            }
+        }
+    }
+
     async updateFailedEmbeddingsList() {
         if (!this.$widget) return;
 

src/public/translations/en/translation.json

@@ -1186,6 +1186,11 @@
     "reprocess_index_started": "Index rebuilding started in the background",
     "reprocess_index_error": "Error rebuilding search index",
 
+    "index_rebuild_progress": "Index Rebuild Progress",
+    "index_rebuilding": "Rebuilding index ({{percentage}}%)",
+    "index_rebuild_complete": "Index rebuild complete",
+    "index_rebuild_status_error": "Error checking index rebuild status",
+
     "embedding_statistics": "Embedding Statistics",
     "total_notes": "Total Notes",
     "processed_notes": "Processed Notes",

src/routes/api/embeddings.ts

@@ -279,6 +279,18 @@ async function rebuildIndex(req: Request, res: Response) {
     };
 }
 
+/**
+ * Get the current index rebuild status
+ */
+async function getIndexRebuildStatus(req: Request, res: Response) {
+    const status = indexService.getIndexRebuildStatus();
+
+    return {
+        success: true,
+        status
+    };
+}
+
 export default {
     findSimilarNotes,
     searchByText,
@@ -290,5 +302,6 @@ export default {
     getFailedNotes,
     retryFailedNote,
     retryAllFailedNotes,
-    rebuildIndex
+    rebuildIndex,
+    getIndexRebuildStatus
 };

src/routes/routes.ts

@@ -384,6 +384,7 @@ function register(app: express.Application) {
     apiRoute(PST, "/api/embeddings/retry/:noteId", embeddingsRoute.retryFailedNote);
     apiRoute(PST, "/api/embeddings/retry-all-failed", embeddingsRoute.retryAllFailedNotes);
     apiRoute(PST, "/api/embeddings/rebuild-index", embeddingsRoute.rebuildIndex);
+    apiRoute(GET, "/api/embeddings/index-rebuild-status", embeddingsRoute.getIndexRebuildStatus);
 
     // LLM chat session management endpoints
     apiRoute(PST, "/api/llm/sessions", llmRoute.createSession);

src/services/llm/embeddings/queue.ts

@@ -8,6 +8,7 @@ import { getNoteEmbeddingContext } from "./content_processing.js";
 import { deleteNoteEmbeddings } from "./storage.js";
 import type { QueueItem } from "./types.js";
 import { getChunkingOperations } from "./chunking_interface.js";
+import indexService from '../index_service.js';
 
 /**
  * Queues a note for embedding update
@@ -176,6 +177,9 @@ export async function processEmbeddingQueue() {
         return;
     }
 
+    // Track successfully processed notes count for progress reporting
+    let processedCount = 0;
+
     for (const note of notes) {
         try {
             const noteData = note as unknown as QueueItem;
@@ -248,6 +252,8 @@ export async function processEmbeddingQueue() {
                     "DELETE FROM embedding_queue WHERE noteId = ?",
                     [noteData.noteId]
                 );
+                // Count as successfully processed
+                processedCount++;
             } else {
                 // If all providers failed, mark as failed but keep in queue
                 await sql.execute(`
@@ -286,4 +292,9 @@ export async function processEmbeddingQueue() {
             }
         }
     }
+
+    // Update the index rebuild progress if any notes were processed
+    if (processedCount > 0) {
+        indexService.updateIndexRebuildProgress(processedCount);
+    }
 }

src/services/llm/index_service.ts

@@ -18,6 +18,7 @@ import { ContextExtractor } from "./context/index.js";
 import eventService from "../events.js";
 import type { NoteEmbeddingContext } from "./embeddings/embeddings_interface.js";
 import type { OptionDefinitions } from "../options_interface.js";
+import sql from "../sql.js";
 
 class IndexService {
     private initialized = false;
@@ -25,6 +26,12 @@ class IndexService {
     private contextExtractor = new ContextExtractor();
     private automaticIndexingInterval?: NodeJS.Timeout;
 
+    // Index rebuilding tracking
+    private indexRebuildInProgress = false;
+    private indexRebuildProgress = 0;
+    private indexRebuildTotal = 0;
+    private indexRebuildCurrent = 0;
+
     // Configuration
     private defaultQueryDepth = 2;
     private maxNotesPerQuery = 10;
@@ -195,6 +202,13 @@ class IndexService {
 
         try {
             this.indexingInProgress = true;
+            this.indexRebuildInProgress = true;
+            this.indexRebuildProgress = 0;
+            this.indexRebuildCurrent = 0;
+
+            // Reset index rebuild progress
+            const totalNotes = await sql.getValue("SELECT COUNT(*) FROM notes WHERE isDeleted = 0") as number;
+            this.indexRebuildTotal = totalNotes;
 
             if (force) {
                 // Force reindexing of all notes
@@ -210,18 +224,55 @@ class IndexService {
                     log.info("Full indexing initiated");
                 } else {
                     log.info(`Skipping full indexing, already at ${stats.percentComplete}% completion`);
+                    this.indexRebuildInProgress = false;
+                    this.indexRebuildProgress = 100;
                 }
             }
 
             return true;
         } catch (error: any) {
             log.error(`Error starting full indexing: ${error.message || "Unknown error"}`);
+            this.indexRebuildInProgress = false;
             return false;
         } finally {
             this.indexingInProgress = false;
         }
     }
 
+    /**
+     * Update index rebuild progress
+     * @param processed - Number of notes processed
+     */
+    updateIndexRebuildProgress(processed: number) {
+        if (!this.indexRebuildInProgress) return;
+
+        this.indexRebuildCurrent += processed;
+
+        if (this.indexRebuildTotal > 0) {
+            this.indexRebuildProgress = Math.min(
+                Math.round((this.indexRebuildCurrent / this.indexRebuildTotal) * 100),
+                100
+            );
+        }
+
+        if (this.indexRebuildCurrent >= this.indexRebuildTotal) {
+            this.indexRebuildInProgress = false;
+            this.indexRebuildProgress = 100;
+        }
+    }
+
+    /**
+     * Get the current index rebuild progress
+     */
+    getIndexRebuildStatus() {
+        return {
+            inProgress: this.indexRebuildInProgress,
+            progress: this.indexRebuildProgress,
+            total: this.indexRebuildTotal,
+            current: this.indexRebuildCurrent
+        };
+    }
+
     /**
      * Run a batch indexing job for a limited number of notes
      * @param batchSize - Maximum number of notes to process