diff --git a/src/entities/note.js b/src/entities/note.js index 0bad7c1c8..6e42ecec1 100644 --- a/src/entities/note.js +++ b/src/entities/note.js @@ -777,7 +777,7 @@ class Note extends Entity { * @returns {NoteRevision[]} */ getRevisions() { - return this.repository.getEntities("SELECT * FROM note_revisions WHERE noteId = ?", [this.noteId]); + return this.repository.getEntities("SELECT * FROM note_revisions WHERE isErased = 0 AND noteId = ?", [this.noteId]); } /** diff --git a/src/public/app/dialogs/options/other.js b/src/public/app/dialogs/options/other.js index 7e498052e..45b271f07 100644 --- a/src/public/app/dialogs/options/other.js +++ b/src/public/app/dialogs/options/other.js @@ -51,6 +51,12 @@ const TPL = ` + +

You can also trigger erasing manually:

+ + + +

@@ -117,6 +123,13 @@ export default class ProtectedSessionOptions { return false; }); + this.$eraseDeletedNotesButton = $("#erase-deleted-notes-now-button"); + this.$eraseDeletedNotesButton.on('click', () => { + server.post('notes/erase-deleted-notes-now').then(() => { + toastService.showMessage("Deleted notes have been erased."); + }); + }); + this.$protectedSessionTimeout = $("#protected-session-timeout-in-seconds"); this.$protectedSessionTimeout.on('change', () => { diff --git a/src/public/app/entities/note_short.js b/src/public/app/entities/note_short.js index dfcf8aafa..6995488ed 100644 --- a/src/public/app/entities/note_short.js +++ b/src/public/app/entities/note_short.js @@ -75,14 +75,16 @@ class NoteShort { this.parentToBranch[parentNoteId] = branchId; } - addChild(childNoteId, branchId) { + addChild(childNoteId, branchId, sort = true) { if (!this.children.includes(childNoteId)) { this.children.push(childNoteId); } this.childToBranch[childNoteId] = branchId; - this.sortChildren(); + if (sort) { + this.sortChildren(); + } } sortChildren() { diff --git a/src/public/app/services/tree_cache.js b/src/public/app/services/tree_cache.js index 1a9c17880..4d5fda4cd 100644 --- a/src/public/app/services/tree_cache.js +++ b/src/public/app/services/tree_cache.js @@ -21,9 +21,6 @@ class TreeCache { async loadInitialTree() { const resp = await server.get('tree'); - // FIXME: we need to do this to cover for ascendants of template notes which are not loaded - await this.loadParents(resp, false); - // clear the cache only directly before adding new content which is important for e.g. switching to protected session /** @type {Object.} */ @@ -44,50 +41,18 @@ class TreeCache { async loadSubTree(subTreeNoteId) { const resp = await server.get('tree?subTreeNoteId=' + subTreeNoteId); - await this.loadParents(resp, true); - this.addResp(resp); return this.notes[subTreeNoteId]; } - async loadParents(resp, additiveLoad) { - const noteIds = new Set(resp.notes.map(note => note.noteId)); - const missingNoteIds = []; - const existingNotes = additiveLoad ? this.notes : {}; - - for (const branch of resp.branches) { - if (!(branch.parentNoteId in existingNotes) && !noteIds.has(branch.parentNoteId) && branch.parentNoteId !== 'none') { - missingNoteIds.push(branch.parentNoteId); - } - } - - for (const attr of resp.attributes) { - if (attr.type === 'relation' && attr.name === 'template' && !(attr.value in existingNotes) && !noteIds.has(attr.value)) { - missingNoteIds.push(attr.value); - } - - if (!(attr.noteId in existingNotes) && !noteIds.has(attr.noteId)) { - missingNoteIds.push(attr.noteId); - } - } - - if (missingNoteIds.length > 0) { - const newResp = await server.post('tree/load', { noteIds: missingNoteIds }); - - resp.notes = resp.notes.concat(newResp.notes); - resp.branches = resp.branches.concat(newResp.branches); - resp.attributes = resp.attributes.concat(newResp.attributes); - - await this.loadParents(resp, additiveLoad); - } - } - addResp(resp) { const noteRows = resp.notes; const branchRows = resp.branches; const attributeRows = resp.attributes; + const noteIdsToSort = new Set(); + for (const noteRow of noteRows) { const {noteId} = noteRow; @@ -154,7 +119,9 @@ class TreeCache { const parentNote = this.notes[branch.parentNoteId]; if (parentNote) { - parentNote.addChild(branch.noteId, branch.branchId); + parentNote.addChild(branch.noteId, branch.branchId, false); + + noteIdsToSort.add(parentNote.noteId); } } @@ -179,6 +146,11 @@ class TreeCache { } } } + + // sort all of them at once, this avoids repeated sorts (#1480) + for (const noteId of noteIdsToSort) { + this.notes[noteId].sortChildren(); + } } async reloadNotes(noteIds) { @@ -190,7 +162,6 @@ class TreeCache { const resp = await server.post('tree/load', { noteIds }); - await this.loadParents(resp, true); this.addResp(resp); const searchNoteIds = []; diff --git a/src/public/app/widgets/note_detail.js b/src/public/app/widgets/note_detail.js index 921a2676d..21bcc8b69 100644 --- a/src/public/app/widgets/note_detail.js +++ b/src/public/app/widgets/note_detail.js @@ -249,13 +249,22 @@ export default class NoteDetailWidget extends TabAwareWidget { this.$widget.find('.note-detail-printable:visible').printThis({ header: $("

").text(this.note && this.note.title).prop('outerHTML'), - footer: "", + footer: ` + + + +`, importCSS: false, loadCSS: [ "libraries/codemirror/codemirror.css", "libraries/ckeditor/ckeditor-content.css", "libraries/ckeditor/ckeditor-content.css", "libraries/bootstrap/css/bootstrap.min.css", + "libraries/katex/katex.min.css", "stylesheets/print.css", "stylesheets/relation_map.css", "stylesheets/themes.css" diff --git a/src/routes/api/notes.js b/src/routes/api/notes.js index 4f03a5223..a839c0bcf 100644 --- a/src/routes/api/notes.js +++ b/src/routes/api/notes.js @@ -193,6 +193,10 @@ function duplicateSubtree(req) { return noteService.duplicateSubtree(noteId, parentNoteId); } +function eraseDeletedNotesNow() { + noteService.eraseDeletedNotesNow(); +} + module.exports = { getNote, updateNote, @@ -204,5 +208,6 @@ module.exports = { setNoteTypeMime, getRelationMap, changeTitle, - duplicateSubtree + duplicateSubtree, + eraseDeletedNotesNow }; diff --git a/src/routes/api/recent_changes.js b/src/routes/api/recent_changes.js index 3f6b9c539..017263405 100644 --- a/src/routes/api/recent_changes.js +++ b/src/routes/api/recent_changes.js @@ -23,7 +23,8 @@ function getRecentChanges(req) { note_revisions.dateCreated AS date FROM note_revisions - JOIN notes USING(noteId)`); + JOIN notes USING(noteId) + WHERE note_revisions.isErased = 0`); for (const noteRevision of noteRevisions) { if (noteCacheService.isInAncestor(noteRevision.noteId, ancestorNoteId)) { diff --git a/src/routes/api/setup.js b/src/routes/api/setup.js index 98cdd6327..d2ec74b2e 100644 --- a/src/routes/api/setup.js +++ b/src/routes/api/setup.js @@ -38,6 +38,8 @@ function saveSyncSeed(req) { }] } + log.info("Saved sync seed."); + sqlInit.createDatabaseForSync(options); } diff --git a/src/routes/api/tree.js b/src/routes/api/tree.js index 4b6572b8f..53578079a 100644 --- a/src/routes/api/tree.js +++ b/src/routes/api/tree.js @@ -5,14 +5,15 @@ const optionService = require('../../services/options'); const treeService = require('../../services/tree'); function getNotesAndBranchesAndAttributes(noteIds) { - noteIds = Array.from(new Set(noteIds)); - const notes = treeService.getNotes(noteIds); + const notes = treeService.getNotesIncludingAscendants(noteIds); - noteIds = notes.map(note => note.noteId); + noteIds = new Set(notes.map(note => note.noteId)); + + sql.fillNoteIdList(noteIds); // joining child note to filter out not completely synchronised notes which would then cause errors later // cannot do that with parent because of root note's 'none' parent - const branches = sql.getManyRows(` + const branches = sql.getRows(` SELECT branches.branchId, branches.noteId, @@ -20,28 +21,45 @@ function getNotesAndBranchesAndAttributes(noteIds) { branches.notePosition, branches.prefix, branches.isExpanded - FROM branches + FROM param_list + JOIN branches ON param_list.paramId = branches.noteId OR param_list.paramId = branches.parentNoteId JOIN notes AS child ON child.noteId = branches.noteId - WHERE branches.isDeleted = 0 - AND (branches.noteId IN (???) OR parentNoteId IN (???))`, noteIds); + WHERE branches.isDeleted = 0`); + + const attributes = sql.getRows(` + SELECT + attributes.attributeId, + attributes.noteId, + attributes.type, + attributes.name, + attributes.value, + attributes.position, + attributes.isInheritable + FROM param_list + JOIN attributes ON attributes.noteId = param_list.paramId + OR (attributes.type = 'relation' AND attributes.value = param_list.paramId) + WHERE attributes.isDeleted = 0`); + + // we don't really care about the direction of the relation + const missingTemplateNoteIds = attributes + .filter(attr => attr.type === 'relation' + && attr.name === 'template' + && !noteIds.has(attr.value)) + .map(attr => attr.value); + + if (missingTemplateNoteIds.length > 0) { + const templateData = getNotesAndBranchesAndAttributes(missingTemplateNoteIds); + + // there are going to be duplicates with simple concatenation, however: + // 1) shouldn't matter for the frontend which will update the entity twice + // 2) there shouldn't be many duplicates. There isn't that many templates + addArrays(notes, templateData.notes); + addArrays(branches, templateData.branches); + addArrays(attributes, templateData.attributes); + } // sorting in memory is faster branches.sort((a, b) => a.notePosition - b.notePosition < 0 ? -1 : 1); - - const attributes = sql.getManyRows(` - SELECT - attributeId, - noteId, - type, - name, - value, - position, - isInheritable - FROM attributes - WHERE isDeleted = 0 - AND (noteId IN (???) OR (type = 'relation' AND value IN (???)))`, noteIds); - - // sorting in memory is faster attributes.sort((a, b) => a.position - b.position < 0 ? -1 : 1); return { @@ -51,6 +69,16 @@ function getNotesAndBranchesAndAttributes(noteIds) { }; } +// should be fast based on https://stackoverflow.com/a/64826145/944162 +// in this case it is assumed that target is potentially much larger than elementsToAdd +function addArrays(target, elementsToAdd) { + while (elementsToAdd.length) { + target.push(elementsToAdd.shift()); + } + + return target; +} + function getTree(req) { const subTreeNoteId = req.query.subTreeNoteId || 'root'; @@ -64,25 +92,8 @@ function getTree(req) { JOIN treeWithDescendants ON branches.parentNoteId = treeWithDescendants.noteId WHERE treeWithDescendants.isExpanded = 1 AND branches.isDeleted = 0 - ), - treeWithDescendantsAndAscendants AS ( - SELECT noteId FROM treeWithDescendants - UNION - SELECT branches.parentNoteId FROM branches - JOIN treeWithDescendantsAndAscendants ON branches.noteId = treeWithDescendantsAndAscendants.noteId - WHERE branches.isDeleted = 0 - AND branches.parentNoteId != ? - ), - treeWithDescendantsAscendantsAndTemplates AS ( - SELECT noteId FROM treeWithDescendantsAndAscendants - UNION - SELECT attributes.value FROM attributes - JOIN treeWithDescendantsAscendantsAndTemplates ON attributes.noteId = treeWithDescendantsAscendantsAndTemplates.noteId - WHERE attributes.isDeleted = 0 - AND attributes.type = 'relation' - AND attributes.name = 'template' ) - SELECT noteId FROM treeWithDescendantsAscendantsAndTemplates`, [subTreeNoteId, subTreeNoteId]); + SELECT noteId FROM treeWithDescendants`, [subTreeNoteId]); noteIds.push(subTreeNoteId); diff --git a/src/routes/routes.js b/src/routes/routes.js index 588111d13..345733007 100644 --- a/src/routes/routes.js +++ b/src/routes/routes.js @@ -154,6 +154,7 @@ function register(app) { route(GET, '/api/notes/:noteId/revisions/:noteRevisionId/download', [auth.checkApiAuthOrElectron], noteRevisionsApiRoute.downloadNoteRevision); apiRoute(PUT, '/api/notes/:noteId/restore-revision/:noteRevisionId', noteRevisionsApiRoute.restoreNoteRevision); apiRoute(POST, '/api/notes/relation-map', notesApiRoute.getRelationMap); + apiRoute(POST, '/api/notes/erase-deleted-notes-now', notesApiRoute.eraseDeletedNotesNow); apiRoute(PUT, '/api/notes/:noteId/change-title', notesApiRoute.changeTitle); apiRoute(POST, '/api/notes/:noteId/duplicate/:parentNoteId', notesApiRoute.duplicateSubtree); diff --git a/src/services/consistency_checks.js b/src/services/consistency_checks.js index 8ab13bee7..32b358bff 100644 --- a/src/services/consistency_checks.js +++ b/src/services/consistency_checks.js @@ -651,7 +651,7 @@ class ConsistencyChecks { // root branch should always be expanded sql.execute("UPDATE branches SET isExpanded = 1 WHERE branchId = 'root'"); - if (this.unrecoveredConsistencyErrors) { + if (!this.unrecoveredConsistencyErrors) { // we run this only if basic checks passed since this assumes basic data consistency this.checkTreeCycles(); diff --git a/src/services/handlers.js b/src/services/handlers.js index 08fc66577..96ee9e0dc 100644 --- a/src/services/handlers.js +++ b/src/services/handlers.js @@ -70,6 +70,10 @@ eventService.subscribe(eventService.ENTITY_CREATED, ({ entityName, entity }) => if (templateNoteContent) { note.setContent(templateNoteContent); } + + note.type = templateNote.type; + note.mime = templateNote.mime; + note.save(); } noteService.duplicateSubtreeWithoutRoot(templateNote.noteId, note.noteId); diff --git a/src/services/note_revisions.js b/src/services/note_revisions.js index cbf97c16a..d1e7f37b4 100644 --- a/src/services/note_revisions.js +++ b/src/services/note_revisions.js @@ -2,6 +2,7 @@ const NoteRevision = require('../entities/note_revision'); const dateUtils = require('../services/date_utils'); +const log = require('../services/log'); /** * @param {Note} note @@ -9,14 +10,21 @@ const dateUtils = require('../services/date_utils'); function protectNoteRevisions(note) { for (const revision of note.getRevisions()) { if (note.isProtected !== revision.isProtected) { - const content = revision.getContent(); + try { + const content = revision.getContent(); - revision.isProtected = note.isProtected; + revision.isProtected = note.isProtected; - // this will force de/encryption - revision.setContent(content); + // this will force de/encryption + revision.setContent(content); - revision.save(); + revision.save(); + } + catch (e) { + log.error("Could not un/protect note revision ID = " + revision.noteRevisionId); + + throw e; + } } } } diff --git a/src/services/notes.js b/src/services/notes.js index de6a10bf5..49ce3acbf 100644 --- a/src/services/notes.js +++ b/src/services/notes.js @@ -183,18 +183,25 @@ function protectNoteRecursively(note, protect, includingSubTree, taskContext) { } function protectNote(note, protect) { - if (protect !== note.isProtected) { - const content = note.getContent(); + try { + if (protect !== note.isProtected) { + const content = note.getContent(); - note.isProtected = protect; + note.isProtected = protect; - // this will force de/encryption - note.setContent(content); + // this will force de/encryption + note.setContent(content); - note.save(); + note.save(); + } + + noteRevisionService.protectNoteRevisions(note); } + catch (e) { + log.error("Could not un/protect note ID = " + note.noteId); - noteRevisionService.protectNoteRevisions(note); + throw e; + } } function findImageLinks(content, foundLinks) { @@ -459,7 +466,7 @@ function saveNoteRevision(note) { const revisionCutoff = dateUtils.utcDateStr(new Date(now.getTime() - noteRevisionSnapshotTimeInterval * 1000)); const existingNoteRevisionId = sql.getValue( - "SELECT noteRevisionId FROM note_revisions WHERE noteId = ? AND utcDateCreated >= ?", [note.noteId, revisionCutoff]); + "SELECT noteRevisionId FROM note_revisions WHERE noteId = ? AND isErased = 0 AND utcDateCreated >= ?", [note.noteId, revisionCutoff]); const msSinceDateCreated = now.getTime() - dateUtils.parseDateTime(note.utcDateCreated).getTime(); @@ -666,8 +673,10 @@ function scanForLinks(note) { } } -function eraseDeletedNotes() { - const eraseNotesAfterTimeInSeconds = optionService.getOptionInt('eraseNotesAfterTimeInSeconds'); +function eraseDeletedNotes(eraseNotesAfterTimeInSeconds = null) { + if (eraseNotesAfterTimeInSeconds === null) { + eraseNotesAfterTimeInSeconds = optionService.getOptionInt('eraseNotesAfterTimeInSeconds'); + } const cutoffDate = new Date(Date.now() - eraseNotesAfterTimeInSeconds * 1000); @@ -717,6 +726,10 @@ function eraseDeletedNotes() { log.info(`Erased notes: ${JSON.stringify(noteIdsToErase)}`); } +function eraseDeletedNotesNow() { + eraseDeletedNotes(0); +} + // do a replace in str - all keys should be replaced by the corresponding values function replaceByMap(str, mapObj) { const re = new RegExp(Object.keys(mapObj).join("|"),"g"); @@ -823,9 +836,9 @@ function getNoteIdMapping(origNote) { sqlInit.dbReady.then(() => { // first cleanup kickoff 5 minutes after startup - setTimeout(cls.wrap(eraseDeletedNotes), 5 * 60 * 1000); + setTimeout(cls.wrap(() => eraseDeletedNotes()), 5 * 60 * 1000); - setInterval(cls.wrap(eraseDeletedNotes), 4 * 3600 * 1000); + setInterval(cls.wrap(() => eraseDeletedNotes()), 4 * 3600 * 1000); }); module.exports = { @@ -839,5 +852,6 @@ module.exports = { duplicateSubtree, duplicateSubtreeWithoutRoot, getUndeletedParentBranches, - triggerNoteTitleChanged + triggerNoteTitleChanged, + eraseDeletedNotesNow }; diff --git a/src/services/sql.js b/src/services/sql.js index fbf0a10dd..1f8ae1be2 100644 --- a/src/services/sql.js +++ b/src/services/sql.js @@ -236,6 +236,29 @@ function transactional(func) { return ret; } +function fillNoteIdList(noteIds, truncate = true) { + if (noteIds.length === 0) { + return; + } + + if (truncate) { + execute("DELETE FROM param_list"); + } + + noteIds = Array.from(new Set(noteIds)); + + if (noteIds.length > 30000) { + fillNoteIdList(noteIds.slice(30000), false); + + noteIds = noteIds.slice(0, 30000); + } + + // doing it manually to avoid this showing up on the sloq query list + const s = stmt(`INSERT INTO param_list VALUES ` + noteIds.map(noteId => `(?)`).join(','), noteIds); + + s.run(noteIds); +} + module.exports = { dbConnection, insert, @@ -253,5 +276,6 @@ module.exports = { executeMany, executeScript, transactional, - upsert + upsert, + fillNoteIdList }; diff --git a/src/services/sql_init.js b/src/services/sql_init.js index c655fcee8..8af9c8f35 100644 --- a/src/services/sql_init.js +++ b/src/services/sql_init.js @@ -40,6 +40,8 @@ async function initDbConnection() { require('./options_init').initStartupOptions(); + sql.execute('CREATE TEMP TABLE "param_list" (`paramId` TEXT NOT NULL PRIMARY KEY)'); + dbReady.resolve(); } diff --git a/src/services/tree.js b/src/services/tree.js index 582517c36..43e6bdc81 100644 --- a/src/services/tree.js +++ b/src/services/tree.js @@ -6,6 +6,41 @@ const Branch = require('../entities/branch'); const entityChangesService = require('./entity_changes.js'); const protectedSessionService = require('./protected_session'); +function getNotesIncludingAscendants(noteIds) { + noteIds = Array.from(new Set(noteIds)); + + sql.fillNoteIdList(noteIds); + + // we return also deleted notes which have been specifically asked for + + const notes = sql.getRows(` + WITH RECURSIVE + treeWithAscendants AS ( + SELECT paramId AS noteId FROM param_list + UNION + SELECT branches.parentNoteId FROM branches + JOIN treeWithAscendants ON branches.noteId = treeWithAscendants.noteId + WHERE branches.isDeleted = 0 + ) + SELECT + noteId, + title, + isProtected, + type, + mime, + isDeleted + FROM notes + JOIN treeWithAscendants USING(noteId)`); + + protectedSessionService.decryptNotes(notes); + + notes.forEach(note => { + note.isProtected = !!note.isProtected + }); + + return notes; +} + function getNotes(noteIds) { // we return also deleted notes which have been specifically asked for const notes = sql.getManyRows(` @@ -190,6 +225,7 @@ function setNoteToParent(noteId, prefix, parentNoteId) { module.exports = { getNotes, + getNotesIncludingAscendants, validateParentChild, sortNotesAlphabetically, setNoteToParent