erasing rows of deleted entities

db/migrations/0175__rename_eraseNotesAfterTimeInSeconds.sql

@@ -0,0 +1 @@
+UPDATE options SET name = 'eraseNotesAfterTimeInSeconds' WHERE name = 'eraseNotesAfterTimeInSeconds';

src/public/app/dialogs/options/other.js

@@ -42,14 +42,14 @@ const TPL = `
 <div>
     <h4>Note erasure timeout</h4>
 
-    <p>Deleted notes are at first only marked as deleted and it is possible to recover them 
+    <p>Deleted notes (and attributes, revisions...) are at first only marked as deleted and it is possible to recover them 
     from Recent Notes dialog. After a period of time, deleted notes are "erased" which means 
     their content is not recoverable anymore. This setting allows you to configure the length 
     of the period between deleting and erasing the note.</p>
 
     <div class="form-group">
-        <label for="erase-notes-after-time-in-seconds">Erase notes after X seconds</label>
+        <label for="erase-entities-after-time-in-seconds">Erase notes after X seconds</label>
-        <input class="form-control" id="erase-notes-after-time-in-seconds" type="number" min="0">
+        <input class="form-control" id="erase-entities-after-time-in-seconds" type="number" min="0">
     </div>
     
     <p>You can also trigger erasing manually:</p>
@@ -111,12 +111,12 @@ export default class ProtectedSessionOptions {
             this.$availableLanguageCodes.text(webContents.session.availableSpellCheckerLanguages.join(', '));
         }
 
-        this.$eraseNotesAfterTimeInSeconds = $("#erase-notes-after-time-in-seconds");
+        this.$eraseEntitiesAfterTimeInSeconds = $("#erase-entities-after-time-in-seconds");
 
-        this.$eraseNotesAfterTimeInSeconds.on('change', () => {
+        this.$eraseEntitiesAfterTimeInSeconds.on('change', () => {
-            const eraseNotesAfterTimeInSeconds = this.$eraseNotesAfterTimeInSeconds.val();
+            const eraseEntitiesAfterTimeInSeconds = this.$eraseEntitiesAfterTimeInSeconds.val();
 
-            server.put('options', { 'eraseNotesAfterTimeInSeconds': eraseNotesAfterTimeInSeconds }).then(() => {
+            server.put('options', { 'eraseEntitiesAfterTimeInSeconds': eraseEntitiesAfterTimeInSeconds }).then(() => {
                 toastService.showMessage("Options change have been saved.");
             });
 
@@ -173,7 +173,7 @@ export default class ProtectedSessionOptions {
         this.$spellCheckEnabled.prop("checked", options['spellCheckEnabled'] === 'true');
         this.$spellCheckLanguageCode.val(options['spellCheckLanguageCode']);
 
-        this.$eraseNotesAfterTimeInSeconds.val(options['eraseNotesAfterTimeInSeconds']);
+        this.$eraseEntitiesAfterTimeInSeconds.val(options['eraseEntitiesAfterTimeInSeconds']);
         this.$protectedSessionTimeout.val(options['protectedSessionTimeout']);
         this.$noteRevisionsTimeInterval.val(options['noteRevisionSnapshotTimeInterval']);
 

src/routes/api/options.js

@@ -7,7 +7,7 @@ const attributes = require('../../services/attributes');
 // options allowed to be updated directly in options dialog
 const ALLOWED_OPTIONS = new Set([
     'username', // not exposed for update (not harmful anyway), needed for reading
-    'eraseNotesAfterTimeInSeconds',
+    'eraseEntitiesAfterTimeInSeconds',
     'protectedSessionTimeout',
     'noteRevisionSnapshotTimeInterval',
     'zoomFactor',

src/services/app_info.js

@@ -4,7 +4,7 @@ const build = require('./build');
 const packageJson = require('../../package');
 const {TRILIUM_DATA_DIR} = require('./data_dir');
 
-const APP_DB_VERSION = 173;
+const APP_DB_VERSION = 175;
 const SYNC_VERSION = 17;
 const CLIPPER_PROTOCOL_VERSION = "1.0";
 

src/services/notes.js

@@ -673,61 +673,90 @@ function scanForLinks(note) {
     }
 }
 
-function eraseDeletedNotes(eraseNotesAfterTimeInSeconds = null) {
+function eraseNotes(noteIdsToErase) {
-    if (eraseNotesAfterTimeInSeconds === null) {
-        eraseNotesAfterTimeInSeconds = optionService.getOptionInt('eraseNotesAfterTimeInSeconds');
-    }
-
-    const cutoffDate = new Date(Date.now() - eraseNotesAfterTimeInSeconds * 1000);
-
-    const noteIdsToErase = sql.getColumn("SELECT noteId FROM notes WHERE isDeleted = 1 AND isErased = 0 AND notes.utcDateModified <= ?", [dateUtils.utcDateStr(cutoffDate)]);
-
     if (noteIdsToErase.length === 0) {
         return;
     }
 
-    // it's better to not use repository for this because:
+    sql.executeMany(`DELETE FROM notes WHERE noteId IN (???)`, noteIdsToErase);
-    // - it would complain about saving protected notes out of protected session
+    sql.executeMany(`UPDATE entity_changes SET isErased = 1 WHERE entityName = 'notes' AND entityId IN (???)`, noteIdsToErase);
-    // - we don't want these changes to be synced (since they are done on all instances anyway)
-    // - we don't want change the hash since this erasing happens on each instance separately
-    //   and changing the hash would fire up the sync errors temporarily
 
-    sql.executeMany(`
+    sql.executeMany(`DELETE FROM note_contents WHERE noteId IN (???)`, noteIdsToErase);
-        UPDATE notes 
+    sql.executeMany(`UPDATE entity_changes SET isErased = 1 WHERE entityName = 'note_contents' AND entityId IN (???)`, noteIdsToErase);
-        SET title = '[erased]',
-            isProtected = 0,
-            isErased = 1
-        WHERE noteId IN (???)`, noteIdsToErase);
 
-    sql.executeMany(`
+    // we also need to erase all "dependent" entities of the erased notes
-        UPDATE note_contents 
+    const branchIdsToErase = sql.getManyRows(`SELECT branchId FROM branches WHERE noteId IN (???)`, noteIdsToErase)
-        SET content = NULL 
+        .map(row => row.branchId);
-        WHERE noteId IN (???)`, noteIdsToErase);
 
-    // deleting first contents since the WHERE relies on isErased = 0
+    eraseBranches(branchIdsToErase);
-    sql.executeMany(`
-        UPDATE note_revision_contents
-        SET content = NULL
-        WHERE noteRevisionId IN 
-            (SELECT noteRevisionId FROM note_revisions WHERE isErased = 0 AND noteId IN (???))`, noteIdsToErase);
 
-    sql.executeMany(`
+    const attributeIdsToErase = sql.getManyRows(`SELECT attributeId FROM attributes WHERE noteId IN (???)`, noteIdsToErase)
-        UPDATE note_revisions 
+        .map(row => row.attributeId);
-        SET isErased = 1,
-            title = NULL
-        WHERE isErased = 0 AND noteId IN (???)`, noteIdsToErase);
 
-    sql.executeMany(`
+    eraseAttributes(attributeIdsToErase);
-        UPDATE attributes 
+
-        SET name = 'deleted',
+    const noteRevisionIdsToErase = sql.getManyRows(`SELECT noteRevisionId FROM note_revisions WHERE noteId IN (???)`, noteIdsToErase)
-            value = ''
+        .map(row => row.noteRevisionId);
-        WHERE noteId IN (???)`, noteIdsToErase);
+
+    eraseNoteRevisions(noteRevisionIdsToErase);
 
     log.info(`Erased notes: ${JSON.stringify(noteIdsToErase)}`);
 }
 
+function eraseBranches(branchIdsToErase) {
+    if (branchIdsToErase.length === 0) {
+        return;
+    }
+
+    sql.executeMany(`DELETE FROM branches WHERE branchId IN (???)`, branchIdsToErase);
+
+    sql.executeMany(`UPDATE entity_changes SET isErased = 1 WHERE entityName = 'branches' AND entityId IN (???)`, branchIdsToErase);
+}
+
+function eraseAttributes(attributeIdsToErase) {
+    if (attributeIdsToErase.length === 0) {
+        return;
+    }
+
+    sql.executeMany(`DELETE FROM attributes WHERE attributeId IN (???)`, attributeIdsToErase);
+
+    sql.executeMany(`UPDATE entity_changes SET isErased = 1 WHERE entityName = 'attributes' AND entityId IN (???)`, attributeIdsToErase);
+}
+
+function eraseNoteRevisions(noteRevisionIdsToErase) {
+    if (noteRevisionIdsToErase.length === 0) {
+        return;
+    }
+
+    sql.executeMany(`DELETE FROM note_revisions WHERE noteRevisionId IN (???)`, noteRevisionIdsToErase);
+    sql.executeMany(`UPDATE entity_changes SET isErased = 1 WHERE entityName = 'note_revisions' AND entityId IN (???)`, noteRevisionIdsToErase);
+
+    sql.executeMany(`DELETE FROM note_revision_contents WHERE noteRevisionId IN (???)`, noteRevisionIdsToErase);
+    sql.executeMany(`UPDATE entity_changes SET isErased = 1 WHERE entityName = 'note_revision_contents' AND entityId IN (???)`, noteRevisionIdsToErase);
+}
+
+function eraseDeletedEntities(eraseEntitiesAfterTimeInSeconds = null) {
+    if (eraseEntitiesAfterTimeInSeconds === null) {
+        eraseEntitiesAfterTimeInSeconds = optionService.getOptionInt('eraseEntitiesAfterTimeInSeconds');
+    }
+
+    const cutoffDate = new Date(Date.now() - eraseEntitiesAfterTimeInSeconds * 1000);
+
+    const noteIdsToErase = sql.getColumn("SELECT noteId FROM notes WHERE isDeleted = 1 AND utcDateModified <= ?", [dateUtils.utcDateStr(cutoffDate)]);
+
+    eraseNotes(noteIdsToErase);
+
+    const branchIdsToErase = sql.getColumn("SELECT branchId FROM branches WHERE isDeleted = 1 AND utcDateModified <= ?", [dateUtils.utcDateStr(cutoffDate)]);
+
+    eraseBranches(branchIdsToErase);
+
+    const attributeIdsToErase = sql.getColumn("SELECT attributeId FROM attributes WHERE isDeleted = 1 AND utcDateModified <= ?", [dateUtils.utcDateStr(cutoffDate)]);
+
+    eraseAttributes(attributeIdsToErase);
+}
+
 function eraseDeletedNotesNow() {
-    eraseDeletedNotes(0);
+    eraseDeletedEntities(0);
 }
 
 // do a replace in str - all keys should be replaced by the corresponding values
@@ -836,9 +865,9 @@ function getNoteIdMapping(origNote) {
 
 sqlInit.dbReady.then(() => {
     // first cleanup kickoff 5 minutes after startup
-    setTimeout(cls.wrap(() => eraseDeletedNotes()), 5 * 60 * 1000);
+    setTimeout(cls.wrap(() => eraseDeletedEntities()), 5 * 60 * 1000);
 
-    setInterval(cls.wrap(() => eraseDeletedNotes()), 4 * 3600 * 1000);
+    setInterval(cls.wrap(() => eraseDeletedEntities()), 4 * 3600 * 1000);
 });
 
 module.exports = {

src/services/options_init.js

@@ -82,7 +82,7 @@ const defaultOptions = [
     { name: 'rightPaneWidth', value: '25', isSynced: false },
     { name: 'rightPaneVisible', value: 'true', isSynced: false },
     { name: 'nativeTitleBarVisible', value: 'false', isSynced: false },
-    { name: 'eraseNotesAfterTimeInSeconds', value: '604800', isSynced: true }, // default is 7 days
+    { name: 'eraseEntitiesAfterTimeInSeconds', value: '604800', isSynced: true }, // default is 7 days
     { name: 'hideArchivedNotes_main', value: 'false', isSynced: false },
     { name: 'hideIncludedImages_main', value: 'true', isSynced: false },
     { name: 'attributeListExpanded', value: 'false', isSynced: false },

src/services/sync_update.js

@@ -1,5 +1,4 @@
 const sql = require('./sql');
-const log = require('./log');
 const entityChangesService = require('./entity_changes.js');
 const eventService = require('./events');
 
@@ -9,22 +8,15 @@ function updateEntity(entityChange, entity, sourceId) {
         return false;
     }
 
-    const {entityName, hash} = entityChange;
+    const updated = entityChange.entityName === 'note_reordering'
-    let updated;
+        ? updateNoteReordering(entityChange, entity, sourceId)
-
+        : updateNormalEntity(entityChange, entity, sourceId);
-    if (entityName === 'note_reordering') {
-        updated = updateNoteReordering(entityChange, entity, sourceId);
-    }
-    else {
-        updated = updateNormalEntity(entityChange, entity, sourceId);
-    }
 
     // currently making exception for protected notes and note revisions because here
     // the title and content are not available decrypted as listeners would expect
-    if (updated &&
+    if (updated && !entity.isProtected) {
-        (!['notes', 'note_contents', 'note_revisions', 'note_revision_contents'].includes(entityName) || !entity.isProtected)) {
         eventService.emit(eventService.ENTITY_SYNCED, {
-            entityName,
+            entityName: entityChange.entityName,
             entity
         });
     }
@@ -42,14 +34,7 @@ function updateNormalEntity(entityChange, entity, sourceId) {
         || hash !== entityChange.hash // sync error, we should still update
     ) {
         if (['note_contents', 'note_revision_contents'].includes(entityChange.entityName)) {
-            // we always use Buffer object which is different from normal saving - there we use simple string type for "string notes"
+            entity.content = handleContent(entity.content);
-            // the problem is that in general it's not possible to whether a note_content is string note or note (syncs can arrive out of order)
-            entity.content = entity.content === null ? null : Buffer.from(entity.content, 'base64');
-
-            if (entity.content && entity.content.byteLength === 0) {
-                // there seems to be a bug which causes empty buffer to be stored as NULL which is then picked up as inconsistency
-                entity.content = "";
-            }
         }
 
         sql.transactional(() => {
@@ -76,6 +61,19 @@ function updateNoteReordering(entityChange, entity, sourceId) {
     return true;
 }
 
+function handleContent(content) {
+    // we always use Buffer object which is different from normal saving - there we use simple string type for "string notes"
+    // the problem is that in general it's not possible to whether a note_content is string note or note (syncs can arrive out of order)
+    content = content === null ? null : Buffer.from(content, 'base64');
+
+    if (content && content.byteLength === 0) {
+        // there seems to be a bug which causes empty buffer to be stored as NULL which is then picked up as inconsistency
+        content = "";
+    }
+
+    return content;
+}
+
 module.exports = {
     updateEntity
 };