diff --git a/.idea/dataSources/a2c75661-f9e2-478f-a69f-6a9409e69997.xml b/.idea/dataSources/a2c75661-f9e2-478f-a69f-6a9409e69997.xml index 2f8739baa..fa34aac35 100644 --- a/.idea/dataSources/a2c75661-f9e2-478f-a69f-6a9409e69997.xml +++ b/.idea/dataSources/a2c75661-f9e2-478f-a69f-6a9409e69997.xml @@ -1,6 +1,6 @@ - + 3.16.1 @@ -17,661 +17,765 @@
-
-
-
-
-
-
+
+ 1 +
+ +
+
+
+ 1 +
+ +
+
+
+
+
1
- +
1
- - +
+ + fts5 + noteId UNINDEXED +title +titleHash UNINDEXED +content +contentHash UNINDEXED + + 1 TEXT|0s 1 - + 2 TEXT|0s 1 - + 3 TEXT|0s 1 - + 4 INT|0s 1 0 - + 5 TEXT|0s 1 "" - + 1 apiTokenId 1 - + apiTokenId 1 sqlite_autoindex_api_tokens_1 - + 1 TEXT|0s 1 - + 2 TEXT|0s 1 - + 3 TEXT|0s 1 - + 4 TEXT|0s 1 - + 5 TEXT|0s 1 '' - + 6 INT|0s 1 0 - + 7 TEXT|0s 1 - + 8 TEXT|0s 1 - + 9 INT|0s 1 - + 10 TEXT|0s 1 "" - + 11 int|0s 0 - + 1 attributeId 1 - + noteId - + name value - + name - + value - + attributeId 1 sqlite_autoindex_attributes_1 - + 1 TEXT|0s 1 - + 2 TEXT|0s 1 - + 3 TEXT|0s 1 - + 4 INTEGER|0s 1 - + 5 TEXT|0s - + 6 BOOLEAN|0s - + 7 INTEGER|0s 1 0 - + 8 TEXT|0s 1 - + 9 TEXT|0s 1 "" - + 10 TEXT|0s 1 '1970-01-01T00:00:00.000Z' - + 1 branchId 1 - + noteId parentNoteId - + noteId - + parentNoteId - + branchId 1 sqlite_autoindex_branches_1 - + 1 TEXT|0s 1 - + 2 TEXT|0s - + 3 TEXT|0s - + 4 TEXT|0s 1 - + 1 eventId 1 - + eventId 1 sqlite_autoindex_event_log_1 - + 1 TEXT|0s 1 - + 2 TEXT|0s 1 - + 3 TEXT|0s 1 - + 4 TEXT|0s 1 - + 5 TEXT|0s 1 "" - + 6 INTEGER|0s 1 0 - + 7 TEXT|0s 1 - + 8 TEXT|0s 1 - + 1 linkId 1 - + noteId - + targetNoteId - + linkId 1 sqlite_autoindex_links_1 - + 1 TEXT|0s 1 - + 2 TEXT|0s 1 - + 3 INT|0s 1 0 - + 4 TEXT|0s NULL - + 5 TEXT|0s 1 "" - + 6 TEXT|0s 1 - '2018-05-08T23:41:15.225Z' - + 7 TEXT|0s 1 - '2018-05-08T23:41:15.225Z' - + 1 noteContentId 1 - + noteId 1 - + noteContentId 1 sqlite_autoindex_note_contents_1 - + + 1 + 1 + + + 2 + + + 1 + k + + 1 + + + k + 1 + sqlite_autoindex_note_fulltext_config_1 + + + 1 + INTEGER|0s + + + 2 + + + 3 + + + 4 + + + 5 + + + 6 + + + id + 1 + + + 1 + INTEGER|0s + + + 2 + BLOB|0s + + + id + 1 + + + 1 + INTEGER|0s + + + 2 + BLOB|0s + + + id + 1 + + + 1 + 1 + + + 2 + 1 + + + 3 + + + 1 + segid +term + + 1 + + + segid +term + 1 + sqlite_autoindex_note_fulltext_idx_1 + + 1 TEXT|0s 1 - + 2 TEXT|0s 1 - + 3 TEXT|0s - + 4 TEXT|0s - + 5 INT|0s 1 0 - + 6 TEXT|0s 1 - + 7 TEXT|0s 1 - + 8 TEXT|0s 1 '' - + 9 TEXT|0s 1 '' - + 10 TEXT|0s 1 "" - + 1 noteRevisionId 1 - + noteId - + dateModifiedFrom - + dateModifiedTo - + noteRevisionId 1 sqlite_autoindex_note_revisions_1 - + 1 TEXT|0s 1 - + 2 TEXT|0s 1 "note" - + 3 INT|0s 1 0 - + 4 TEXT|0s 1 'text' - + 5 TEXT|0s 1 'text/html' - + 6 TEXT|0s 1 "" - + 7 INT|0s 1 0 - + 8 TEXT|0s 1 - + 9 TEXT|0s 1 - + 1 noteId 1 - + noteId 1 sqlite_autoindex_notes_1 - + 1 TEXT|0s 1 - + 2 TEXT|0s - + 3 INT|0s - + 4 INTEGER|0s 1 0 - + 5 TEXT|0s 1 "" - + 6 TEXT|0s 1 '1970-01-01T00:00:00.000Z' - + 1 name 1 - + name 1 sqlite_autoindex_options_1 - + 1 TEXT|0s 1 - + 2 TEXT|0s 1 - + 3 TEXT|0s 1 "" - + 4 TEXT|0s 1 - + 5 INT|0s - + 1 branchId 1 - + branchId 1 sqlite_autoindex_recent_notes_1 - + 1 TEXT|0s 1 - + 2 TEXT|0s 1 - + 1 sourceId 1 - + sourceId 1 sqlite_autoindex_source_ids_1 - + 1 text|0s - + 2 text|0s - + 3 text|0s - + 4 integer|0s - + 5 text|0s - + 1 - + 2 - + 1 INTEGER|0s 1 1 - + 2 TEXT|0s 1 - + 3 TEXT|0s 1 - + 4 TEXT|0s 1 - + 5 TEXT|0s 1 - + entityName entityId 1 - + syncDate - + id 1 diff --git a/db/migrations/0127__add_note_fulltext.sql b/db/migrations/0127__add_note_fulltext.sql new file mode 100644 index 000000000..d9910230a --- /dev/null +++ b/db/migrations/0127__add_note_fulltext.sql @@ -0,0 +1 @@ +CREATE VIRTUAL TABLE note_fulltext USING fts5(noteId UNINDEXED, title, titleHash UNINDEXED, content, contentHash UNINDEXED); diff --git a/db/migrations/0128__fill_note_fulltext.js b/db/migrations/0128__fill_note_fulltext.js new file mode 100644 index 000000000..1709d10fb --- /dev/null +++ b/db/migrations/0128__fill_note_fulltext.js @@ -0,0 +1,10 @@ +const repository = require('../../src/services/repository'); +const noteFulltextService = require('../../src/services/note_fulltext'); + +module.exports = async () => { + const notes = await repository.getEntities('SELECT * FROM notes WHERE isDeleted = 0 AND isProtected = 0'); + + for (const note of notes) { + await noteFulltextService.updateNoteFulltext(note); + } +}; \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index bd02e3e31..8403dc408 100644 --- a/package-lock.json +++ b/package-lock.json @@ -430,8 +430,7 @@ "@types/node": { "version": "10.12.21", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.21.tgz", - "integrity": "sha512-CBgLNk4o3XMnqMc0rhb6lc77IwShMEglz05deDcn2lQxyXEZivfwgYJu7SMha9V5XcrP6qZuevTHV/QrN2vjKQ==", - "dev": true + "integrity": "sha512-CBgLNk4o3XMnqMc0rhb6lc77IwShMEglz05deDcn2lQxyXEZivfwgYJu7SMha9V5XcrP6qZuevTHV/QrN2vjKQ==" }, "abab": { "version": "2.0.0", @@ -1111,6 +1110,11 @@ } } }, + "boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=" + }, "boxen": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/boxen/-/boxen-1.3.0.tgz", @@ -1563,6 +1567,29 @@ "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", "dev": true }, + "cheerio": { + "version": "1.0.0-rc.2", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.2.tgz", + "integrity": "sha1-S59TqBsn5NXawxwP/Qz6A8xoMNs=", + "requires": { + "css-select": "~1.2.0", + "dom-serializer": "~0.1.0", + "entities": "~1.1.1", + "htmlparser2": "^3.9.1", + "lodash": "^4.15.0", + "parse5": "^3.0.1" + }, + "dependencies": { + "parse5": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz", + "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==", + "requires": { + "@types/node": "*" + } + } + } + }, "chownr": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.1.tgz", @@ -1994,6 +2021,22 @@ "integrity": "sha1-ojD2T1aDEOFJgAmUB5DsmVRbyn4=", "dev": true }, + "css-select": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", + "requires": { + "boolbase": "~1.0.0", + "css-what": "2.1", + "domutils": "1.5.1", + "nth-check": "~1.0.1" + } + }, + "css-what": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz", + "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg==" + }, "cssom": { "version": "0.3.4", "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.4.tgz", @@ -2514,11 +2557,25 @@ "esutils": "^2.0.2" } }, + "dom-serializer": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz", + "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==", + "requires": { + "domelementtype": "^1.3.0", + "entities": "^1.1.1" + } + }, "dom-walk": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/dom-walk/-/dom-walk-0.1.1.tgz", "integrity": "sha1-ZyIm3HTI95mtNTB9+TaroRrNYBg=" }, + "domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==" + }, "domexception": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/domexception/-/domexception-1.0.1.tgz", @@ -2527,6 +2584,23 @@ "webidl-conversions": "^4.0.2" } }, + "domhandler": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", + "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", + "requires": { + "domelementtype": "1" + } + }, + "domutils": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", + "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", + "requires": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, "dont-sniff-mimetype": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/dont-sniff-mimetype/-/dont-sniff-mimetype-1.0.0.tgz", @@ -5470,6 +5544,11 @@ "integrity": "sha1-ieJdtgS3Jcj1l2//Ct3JIbgopac=", "dev": true }, + "he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==" + }, "helmet": { "version": "3.15.1", "resolved": "https://registry.npmjs.org/helmet/-/helmet-3.15.1.tgz", @@ -5557,6 +5636,41 @@ "whatwg-encoding": "^1.0.1" } }, + "html2plaintext": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/html2plaintext/-/html2plaintext-2.1.2.tgz", + "integrity": "sha512-/7rk161q0RFtQhu0F7oU7MFUtqjm2qBrVfoS8EOaHSdRNt72CNNYSV1/wN+TfO2GhgLQdIjPctmiWPX3oRcNFQ==", + "requires": { + "cheerio": "1.0.0-rc.2", + "he": "1.2.0", + "plumb": "0.1.0" + } + }, + "htmlparser2": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz", + "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==", + "requires": { + "domelementtype": "^1.3.1", + "domhandler": "^2.3.0", + "domutils": "^1.5.1", + "entities": "^1.1.1", + "inherits": "^2.0.1", + "readable-stream": "^3.1.1" + }, + "dependencies": { + "readable-stream": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.2.0.tgz", + "integrity": "sha512-RV20kLjdmpZuTF1INEb9IA3L68Nmi+Ri7ppZqo78wj//Pn62fCoJyV9zalccNzDD/OuJpMG4f+pfMl8+L6QdGw==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, "http-cache-semantics": { "version": "3.8.1", "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz", @@ -8038,6 +8152,14 @@ "set-blocking": "~2.0.0" } }, + "nth-check": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", + "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", + "requires": { + "boolbase": "~1.0.0" + } + }, "nugget": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/nugget/-/nugget-2.0.1.tgz", @@ -8771,6 +8893,11 @@ } } }, + "plumb": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/plumb/-/plumb-0.1.0.tgz", + "integrity": "sha1-TFd5ClCWkoMv2/EN+t3XlIxctXQ=" + }, "plur": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/plur/-/plur-3.0.1.tgz", @@ -10850,6 +10977,11 @@ "escape-string-regexp": "^1.0.2" } }, + "striptags": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/striptags/-/striptags-3.1.1.tgz", + "integrity": "sha1-yMPn/db7S7OjKjt1LltePjgJPr0=" + }, "sum-up": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sum-up/-/sum-up-1.0.3.tgz", diff --git a/package.json b/package.json index 975f4a263..5a7dabef3 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "get-port": "4.2.0", "helmet": "3.15.1", "html": "1.0.0", + "html2plaintext": "^2.1.2", "image-type": "3.0.0", "imagemin": "6.1.0", "imagemin-giflossy": "5.1.10", diff --git a/src/entities/note.js b/src/entities/note.js index a206b5caa..2855d63a4 100644 --- a/src/entities/note.js +++ b/src/entities/note.js @@ -7,6 +7,7 @@ const protectedSessionService = require('../services/protected_session'); const repository = require('../services/repository'); const sql = require('../services/sql'); const dateUtils = require('../services/date_utils'); +const noteFulltextService = require('../services/note_fulltext'); const LABEL = 'label'; const LABEL_DEFINITION = 'label-definition'; @@ -687,6 +688,10 @@ class Note extends Entity { delete pojo.titleCipherText; delete pojo.noteContent; } + + async afterSaving() { + noteFulltextService.triggerNoteFulltextUpdate(this.noteId); + } } module.exports = Note; \ No newline at end of file diff --git a/src/entities/note_content.js b/src/entities/note_content.js index bb934b5e4..fd4a44365 100644 --- a/src/entities/note_content.js +++ b/src/entities/note_content.js @@ -4,6 +4,7 @@ const Entity = require('./entity'); const protectedSessionService = require('../services/protected_session'); const repository = require('../services/repository'); const dateUtils = require('../services/date_utils'); +const noteFulltextService = require('../services/note_fulltext'); /** * This represents a Note which is a central object in the Trilium Notes project. @@ -91,6 +92,10 @@ class NoteContent extends Entity { delete pojo.isContentAvailable; delete pojo.contentCipherText; } + + async afterSaving() { + noteFulltextService.triggerNoteFulltextUpdate(this.noteId); + } } module.exports = NoteContent; \ No newline at end of file diff --git a/src/services/app_info.js b/src/services/app_info.js index 7396fceb4..7d52561fa 100644 --- a/src/services/app_info.js +++ b/src/services/app_info.js @@ -4,8 +4,8 @@ const build = require('./build'); const packageJson = require('../../package'); const {TRILIUM_DATA_DIR} = require('./data_dir'); -const APP_DB_VERSION = 126; -const SYNC_VERSION = 6; +const APP_DB_VERSION = 128; +const SYNC_VERSION = 7; module.exports = { appVersion: packageJson.version, diff --git a/src/services/note_fulltext.js b/src/services/note_fulltext.js new file mode 100644 index 000000000..8d31b36df --- /dev/null +++ b/src/services/note_fulltext.js @@ -0,0 +1,60 @@ +const sql = require('./sql'); +const repository = require('./repository'); +const html2plaintext = require('html2plaintext'); + +const noteIdQueue = []; + +async function updateNoteFulltext(note) { + if (note.isDeleted || note.isProtected || note.hasLabel('archived')) { + await sql.execute(`DELETE + FROM note_fulltext + WHERE noteId = ?`, [note.noteId]); + } else { + let content = null; + let contentHash = null; + + if (['text', 'code'].includes(note.type)) { + const noteContent = await note.getNoteContent(); + content = noteContent.content; + + if (note.type === 'text' && note.mime === 'text/html') { + content = html2plaintext(content); + } + + contentHash = noteContent.hash; + } + + // optimistically try to update first ... + const res = await sql.execute(`UPDATE note_fulltext title = ?, titleHash = ?, content = ?, contentHash = ? WHERE noteId = ?`, [note.title, note.hash, content, contentHash, note.noteId]); + + // ... and insert only when the update did not work + if (res.stmt.changes === 0) { + await sql.execute(`INSERT INTO note_fulltext (title, titleHash, content, contentHash, noteId) + VALUES (?, ?, ?, ?, ?)`, [note.title, note.hash, content, contentHash, note.noteId]); + } + } +} + +async function triggerNoteFulltextUpdate(noteId) { + if (!noteIdQueue.includes(noteId)) { + noteIdQueue.push(noteId); + } + + while (noteIdQueue.length > 0) { + await sql.transactional(async () => { + if (noteIdQueue.length === 0) { + return; + } + + const noteId = noteIdQueue.shift(); + const note = await repository.getNote(noteId); + + await updateNoteFulltext(note); + }); + } +} + +module.exports = { + triggerNoteFulltextUpdate, + updateNoteFulltext +}; \ No newline at end of file diff --git a/src/services/repository.js b/src/services/repository.js index 967bdd19d..383ff7864 100644 --- a/src/services/repository.js +++ b/src/services/repository.js @@ -126,6 +126,10 @@ async function updateEntity(entity) { await eventService.emit(entity.isDeleted ? eventService.ENTITY_DELETED : eventService.ENTITY_CHANGED, eventPayload); } } + + if (entity.afterSaving) { + await entity.afterSaving(); + } }); } diff --git a/src/services/sync_update.js b/src/services/sync_update.js index ff439a7bd..81b9a2465 100644 --- a/src/services/sync_update.js +++ b/src/services/sync_update.js @@ -3,6 +3,7 @@ const log = require('./log'); const eventLogService = require('./event_log'); const syncTableService = require('./sync_table'); const eventService = require('./events'); +const noteFulltextService = require('../services/note_fulltext'); async function updateEntity(sync, entity, sourceId) { const {entityName} = sync; @@ -61,6 +62,8 @@ async function updateNote(entity, sourceId) { await syncTableService.addNoteSync(entity.noteId, sourceId); }); + noteFulltextService.triggerNoteFulltextUpdate(entity.noteId); + log.info("Update/sync note " + entity.noteId); } } @@ -75,6 +78,8 @@ async function updateNoteContent(entity, sourceId) { await sql.replace("note_contents", entity); await syncTableService.addNoteContentSync(entity.noteContentId, sourceId); + + noteFulltextService.triggerNoteFulltextUpdate(entity.noteId); }); log.info("Update/sync note content " + entity.noteContentId);