Tools/Notes
1
0
Fork 0
mirror of https://github.com/TriliumNext/Notes.git synced 2025-10-31 04:51:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

note cache refactoring

Browse Source
This commit is contained in:
zadam 2020-05-13 14:42:16 +02:00
parent b07accfd9d
commit 7992f32d34
7 changed files with 210 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
package-lock.json generated
View File

@ -1,6 +1,6 @@
{
"name": "trilium",
"version": "0.42.1",
"version": "0.42.2",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
@ -2218,9 +2218,9 @@
}
},
"cli-spinners": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.2.0.tgz",
"integrity": "sha512-tgU3fKwzYjiLEQgPMD9Jt+JjHVL9kW93FiIMX/l7rivvOD4/LL0Mf7gda3+4U2KJBloybwgj5KEoQgGRioMiKQ==",
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.3.0.tgz",
"integrity": "sha512-Xs2Hf2nzrvJMFKimOR7YR0QwZ8fc0u98kdtwN1eNAZzNQgH3vK2pXzff6GJtKh7S5hoJ87ECiAiZFS2fb5Ii2w==",
"dev": true
},
"cli-table3": {
@ -3802,9 +3802,9 @@
}
},
"electron-rebuild": {
"version": "1.10.1",
"resolved": "https://registry.npmjs.org/electron-rebuild/-/electron-rebuild-1.10.1.tgz",
"integrity": "sha512-KSqp0Xiu7CCvKL2aEdPp/vNe2Rr11vaO8eM/wq9gQJTY02UjtAJ3l7WLV7Mf8oR+UJReJO8SWOWs/FozqK8ggA==",
"version": "1.11.0",
"resolved": "https://registry.npmjs.org/electron-rebuild/-/electron-rebuild-1.11.0.tgz",
"integrity": "sha512-cn6AqZBQBVtaEyj5jZW1/LOezZZ22PA1HvhEP7asvYPJ8PDF4i4UFt9be4i9T7xJKiSiomXvY5Fd+dSq3FXZxA==",
"dev": true,
"requires": {
"colors": "^1.3.3",
@ -3877,9 +3877,9 @@
}
},
"yargs": {
"version": "14.2.2",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-14.2.2.tgz",
"integrity": "sha512-/4ld+4VV5RnrynMhPZJ/ZpOCGSCeghMykZ3BhdFBDa9Wy/RH6uEGNWDJog+aUlq+9OM1CFTgtYRW5Is1Po9NOA==",
"version": "14.2.3",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-14.2.3.tgz",
"integrity": "sha512-ZbotRWhF+lkjijC/VhmOT9wSgyBQ7+zr13+YLkhfsSiTriYsMzkTUFP18pFhWwBeMa5gUc1MzbhrO6/VB7c9Xg==",
"dev": true,
"requires": {
"cliui": "^5.0.0",
@ -3892,13 +3892,13 @@
"string-width": "^3.0.0",
"which-module": "^2.0.0",
"y18n": "^4.0.0",
"yargs-parser": "^15.0.0"
"yargs-parser": "^15.0.1"
}
},
"yargs-parser": {
"version": "15.0.0",
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-15.0.0.tgz",
"integrity": "sha512-xLTUnCMc4JhxrPEPUYD5IBR1mWCK/aT6+RJ/K29JY2y1vD+FhtgKK0AXRWvI262q3QSffAQuTouFIKUuHX89wQ==",
"version": "15.0.1",
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-15.0.1.tgz",
"integrity": "sha512-0OAMV2mAZQrs3FkNpDQcBk1x5HXb8X4twADss4S0Iuk+2dGnLOE/fRHrsYm542GduMveyA77OF4wrNJuanRCWw==",
"dev": true,
"requires": {
"camelcase": "^5.0.0",
@ -9929,9 +9929,9 @@
}
},
"rxjs": {
"version": "6.5.4",
"resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.4.tgz",
"integrity": "sha512-naMQXcgEo3csAEGvw/NydRA0fuS2nDZJiw1YUWFKU7aPPAPGZEsD4Iimit96qwCieH6y614MCLYwdkrWx7z/7Q==",
"version": "6.5.5",
"resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.5.tgz",
"integrity": "sha512-WfQI+1gohdf0Dai/Bbmk5L5ItH5tYqm3ki2c5GdWhKjalzjg93N3avFjVStyZZz+A2Em+ZxKH5bNghw9UeylGQ==",
"dev": true,
"requires": {
"tslib": "^1.9.0"

2
package.json
View File

@ -81,7 +81,7 @@
"electron": "9.0.0-beta.24",
"electron-builder": "22.6.0",
"electron-packager": "14.2.1",
"electron-rebuild": "1.10.1",
"electron-rebuild": "1.11.0",
"jsdoc": "3.6.4",
"lorem-ipsum": "2.0.3",
"webpack": "5.0.0-beta.16",

14
src/entities/attribute.js
View File

@ -8,13 +8,13 @@ const sql = require('../services/sql');
/**
* Attribute is key value pair owned by a note.
*
* @property {string} attributeId
* @property {string} noteId
* @property {string} type
* @property {string} name
* @property {string} attributeId - immutable
* @property {string} noteId - immutable
* @property {string} type - immutable
* @property {string} name - immutable
* @property {string} value
* @property {int} position
* @property {boolean} isInheritable
* @property {boolean} isInheritable - immutable
* @property {boolean} isDeleted
* @property {string|null} deleteId - ID identifying delete transaction
* @property {string} utcDateCreated
@ -108,14 +108,14 @@ class Attribute extends Entity {
delete pojo.__note;
}
createClone(type, name, value) {
createClone(type, name, value, isInheritable) {
return new Attribute({
noteId: this.noteId,
type: type,
name: name,
value: value,
position: this.position,
isInheritable: this.isInheritable,
isInheritable: isInheritable,
isDeleted: false,
utcDateCreated: this.utcDateCreated,
utcDateModified: this.utcDateModified

6
src/entities/branch.js
View File

@ -9,9 +9,9 @@ const sql = require('../services/sql');
* Branch represents note's placement in the tree - it's essentially pair of noteId and parentNoteId.
* Each note can have multiple (at least one) branches, meaning it can be placed into multiple places in the tree.
*
* @property {string} branchId - primary key
* @property {string} noteId
* @property {string} parentNoteId
* @property {string} branchId - primary key, immutable
* @property {string} noteId - immutable
* @property {string} parentNoteId - immutable
* @property {int} notePosition
* @property {string} prefix
* @property {boolean} isExpanded

5
src/routes/api/attributes.js
View File

@ -98,10 +98,11 @@ async function updateNoteAttributes(req) {
if (attribute.type !== attributeEntity.type
|| attribute.name !== attributeEntity.name
|| (attribute.type === 'relation' && attribute.value !== attributeEntity.value)) {
|| (attribute.type === 'relation' && attribute.value !== attributeEntity.value)
|| attribute.isInheritable !== attributeEntity.isInheritable) {
if (attribute.type !== 'relation' || !!attribute.value.trim()) {
const newAttribute = attributeEntity.createClone(attribute.type, attribute.name, attribute.value);
const newAttribute = attributeEntity.createClone(attribute.type, attribute.name, attribute.value, attribute.isInheritable);
await newAttribute.save();
}

2
src/routes/api/similar_notes.js
View File

@ -12,7 +12,7 @@ async function getSimilarNotes(req) {
return [404, `Note ${noteId} not found.`];
}
const results = await noteCacheService.findSimilarNotes(note.title);
const results = await noteCacheService.findSimilarNotes(noteId);
return results
.filter(note => note.noteId !== noteId);

301
src/services/note_cache.js
View File

@ -35,46 +35,75 @@ class Note {
this.children = [];
/** @param {Attribute[]} */
this.ownedAttributes = [];
/** @param {Attribute[]|null} */
this.attributeCache = null;
/** @param {Attribute[]|null} */
this.templateAttributeCache = null;
/** @param {Attribute[]|null} */
this.inheritableAttributeCache = null;
/** @param {string|null} */
this.fulltextCache = null;
}
/** @return {Attribute[]} */
get attributes() {
if (!(this.noteId in noteAttributeCache)) {
const attrArrs = [
this.ownedAttributes
];
for (const templateAttr of this.ownedAttributes.filter(oa => oa.type === 'relation' && oa.name === 'template')) {
const templateNote = notes[templateAttr.value];
if (templateNote) {
attrArrs.push(templateNote.attributes);
}
}
if (!this.attributeCache) {
const parentAttributes = this.ownedAttributes.slice();
if (this.noteId !== 'root') {
for (const parentNote of this.parents) {
attrArrs.push(parentNote.inheritableAttributes);
parentAttributes.push(...parentNote.inheritableAttributes);
}
}
noteAttributeCache[this.noteId] = attrArrs.flat();
const templateAttributes = [];
for (const ownedAttr of parentAttributes) { // parentAttributes so we process also inherited templates
if (ownedAttr.type === 'relation' && ownedAttr.name === 'template') {
const templateNote = notes[ownedAttr.value];
if (templateNote) {
templateAttributes.push(...templateNote.attributes);
}
}
}
return noteAttributeCache[this.noteId];
this.attributeCache = parentAttributes.concat(templateAttributes);
this.inheritableAttributeCache = [];
this.templateAttributeCache = [];
for (const attr of this.attributeCache) {
if (attr.isInheritable) {
this.inheritableAttributeCache.push(attr);
}
addSubTreeNoteIdsTo(noteIdSet) {
noteIdSet.add(this.noteId);
for (const child of this.children) {
child.addSubTreeNoteIdsTo(noteIdSet);
if (attr.type === 'relation' && attr.name === 'template') {
this.templateAttributeCache.push(attr);
}
}
}
return this.attributeCache;
}
/** @return {Attribute[]} */
get inheritableAttributes() {
return this.attributes.filter(attr => attr.isInheritable);
if (!this.inheritableAttributeCache) {
this.attributes; // will refresh also this.inheritableAttributeCache
}
return this.inheritableAttributeCache;
}
/** @return {Attribute[]} */
get templateAttributes() {
if (!this.templateAttributeCache) {
this.attributes; // will refresh also this.templateAttributeCache
}
return this.templateAttributeCache;
}
hasAttribute(type, name) {
@ -94,6 +123,63 @@ class Note {
resortParents() {
this.parents.sort((a, b) => a.hasInheritableOwnedArchivedLabel ? 1 : -1);
}
get fulltext() {
if (!this.fulltextCache) {
this.fulltextCache = this.title.toLowerCase();
for (const attr of this.attributes) {
// it's best to use space as separator since spaces are filtered from the search string by the tokenization into words
this.fulltextCache += ' ' + attr.name.toLowerCase();
if (attr.value) {
this.fulltextCache += ' ' + attr.value.toLowerCase();
}
}
}
return this.fulltextCache;
}
invalidateThisCache() {
this.fulltextCache = null;
this.attributeCache = null;
this.templateAttributeCache = null;
this.inheritableAttributeCache = null;
}
invalidateSubtreeCaches() {
this.invalidateThisCache();
for (const childNote of this.children) {
childNote.invalidateSubtreeCaches();
}
for (const templateAttr of this.templateAttributes) {
const targetNote = templateAttr.targetNote;
if (targetNote) {
targetNote.invalidateSubtreeCaches();
}
}
}
invalidateSubtreeFulltext() {
this.fulltextCache = null;
for (const childNote of this.children) {
childNote.invalidateSubtreeFulltext();
}
for (const templateAttr of this.templateAttributes) {
const targetNote = templateAttr.targetNote;
if (targetNote) {
targetNote.invalidateSubtreeFulltext();
}
}
}
}
class Branch {
@ -137,47 +223,16 @@ class Attribute {
/** @param {boolean} */
this.isInheritable = !!row.isInheritable;
}
}
/** @type {Object.<String, String>} */
let fulltext = {};
/** @type {Object.<String, AttributeMeta>} */
let attributeMetas = {};
class AttributeMeta {
constructor(attribute) {
this.type = attribute.type;
this.name = attribute.name;
this.isInheritable = attribute.isInheritable;
this.attributeIds = new Set(attribute.attributeId);
get isAffectingSubtree() {
return this.isInheritable
|| (this.type === 'relation' && this.name === 'template');
}
addAttribute(attribute) {
this.attributeIds.add(attribute.attributeId);
this.isInheritable = this.isInheritable || attribute.isInheritable;
get targetNote() {
if (this.type === 'relation') {
return notes[this.value];
}
updateAttribute(attribute) {
if (attribute.isDeleted) {
this.attributeIds.delete(attribute.attributeId);
}
else {
this.attributeIds.add(attribute.attributeId);
}
this.isInheritable = !!this.attributeIds.find(attributeId => attributes[attributeId].isInheritable);
}
}
function addToAttributeMeta(attribute) {
const key = `${attribute.type}-${attribute.name}`;
if (!(key in attributeMetas)) {
attributeMetas[key] = new AttributeMeta(attribute);
}
else {
attributeMetas[key].addAttribute(attribute);
}
}
@ -186,9 +241,6 @@ let loadedPromiseResolve;
/** Is resolved after the initial load */
let loadedPromise = new Promise(res => loadedPromiseResolve = res);
// key is 'childNoteId-parentNoteId' as a replacement for branchId which we don't use here
let prefixes = {};
async function getMappedRows(query, cb) {
const map = {};
const results = await sql.getRows(query, []);
@ -202,17 +254,6 @@ async function getMappedRows(query, cb) {
return map;
}
function updateFulltext(note) {
let ft = note.title.toLowerCase();
for (const attr of note.attributes) {
ft += '|' + attr.name.toLowerCase();
ft += '|' + attr.value.toLowerCase();
}
fulltext[note.noteId] = ft;
}
async function load() {
notes = await getMappedRows(`SELECT noteId, title, isProtected FROM notes WHERE isDeleted = 0`,
row => new Note(row));
@ -225,8 +266,6 @@ async function load() {
for (const attr of Object.values(attributes)) {
notes[attr.noteId].ownedAttributes.push(attr);
addToAttributeMeta(attributes);
}
for (const branch of Object.values(branches)) {
@ -250,10 +289,6 @@ async function load() {
await decryptProtectedNotes();
}
for (const note of Object.values(notes)) {
updateFulltext(note);
}
loaded = true;
loadedPromiseResolve();
}
@ -325,38 +360,21 @@ function highlightResults(results, allTokens) {
* Returns noteIds which have at least one matching tokens
*
* @param tokens
* @return {Set<String>}
* @return {String[]}
*/
function getCandidateNotes(tokens) {
const candidateNoteIds = new Set();
const candidateNotes = [];
for (const note of Object.values(notes)) {
for (const token of tokens) {
for (const noteId in fulltext) {
if (!fulltext[noteId].includes(token)) {
continue;
if (note.fulltext.includes(token)) {
candidateNotes.push(note);
break;
}
}
}
candidateNoteIds.add(noteId);
const note = notes[noteId];
const inheritableAttrs = note.ownedAttributes.filter(attr => attr.isInheritable);
searchingAttrs:
// for matching inheritable attributes, include the whole note subtree to the candidates
for (const attr of inheritableAttrs) {
const lcName = attr.name.toLowerCase();
const lcValue = attr.value.toLowerCase();
for (const token of tokens) {
if (lcName.includes(token) || lcValue.includes(token)) {
note.addSubTreeNoteIdsTo(candidateNoteIds);
break searchingAttrs;
}
}
}
}
}
return candidateNoteIds;
return candidateNotes;
}
async function findNotes(query) {
@ -370,18 +388,16 @@ async function findNotes(query) {
.split(/[ -]/)
.filter(token => token !== '/'); // '/' is used as separator
const candidateNoteIds = getCandidateNotes(allTokens);
const candidateNotes = getCandidateNotes(allTokens);
// now we have set of noteIds which match at least one token
let results = [];
const tokens = allTokens.slice();
for (const noteId of candidateNoteIds) {
const note = notes[noteId];
for (const note of candidateNotes) {
// autocomplete should be able to find notes by their noteIds as well (only leafs)
if (noteId === query) {
if (note.noteId === query) {
search(note, [], [], results);
continue;
}
@ -415,7 +431,7 @@ async function findNotes(query) {
if (foundTokens.length > 0) {
const remainingTokens = tokens.filter(token => !foundTokens.includes(token));
search(parentNote, remainingTokens, [noteId], results);
search(parentNote, remainingTokens, [note.noteId], results);
}
}
}
@ -678,11 +694,11 @@ function getNotePath(noteId) {
}
}
function evaluateSimilarity(text, note, results) {
let coeff = stringSimilarity.compareTwoStrings(text, note.title);
function evaluateSimilarity(sourceNote, candidateNote, results) {
let coeff = stringSimilarity.compareTwoStrings(sourceNote.fulltext, candidateNote.fulltext);
if (coeff > 0.4) {
const notePath = getSomePath(note);
const notePath = getSomePath(candidateNote);
// this takes care of note hoisting
if (!notePath) {
@ -693,7 +709,7 @@ function evaluateSimilarity(text, note, results) {
coeff -= 0.2; // archived penalization
}
results.push({coeff, notePath, noteId: note.noteId});
results.push({coeff, notePath, noteId: candidateNote.noteId});
}
}
@ -707,16 +723,22 @@ function setImmediatePromise() {
});
}
async function findSimilarNotes(title) {
async function findSimilarNotes(noteId) {
const results = [];
let i = 0;
const origNote = notes[noteId];
if (!origNote) {
return [];
}
for (const note of Object.values(notes)) {
if (note.isProtected && !note.isDecrypted) {
continue;
}
evaluateSimilarity(title, note, results);
evaluateSimilarity(origNote, note, results);
i++;
@ -744,9 +766,12 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED
delete notes[noteId];
}
else if (noteId in notes) {
const note = notes[noteId];
// we can assume we have protected session since we managed to update
notes[noteId].title = entity.title;
notes[noteId].isDecrypted = true;
note.title = entity.title;
note.isDecrypted = true;
note.fulltextCache = null;
}
else {
notes[noteId] = new Note(entity);
@ -760,6 +785,10 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED
if (childNote) {
childNote.parents = childNote.parents.filter(parent => parent.noteId !== parentNoteId);
if (childNote.parents.length > 0) {
childNote.invalidateSubtreeCaches();
}
}
const parentNote = notes[parentNoteId];
@ -787,30 +816,46 @@ eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED
}
else if (entityName === 'attributes') {
const {attributeId, noteId} = entity;
const note = notes[noteId];
const attr = attributes[attributeId];
if (entity.isDeleted) {
const note = notes[noteId];
if (note) {
if (note && attr) {
note.ownedAttributes = note.ownedAttributes.filter(attr => attr.attributeId !== attributeId);
if (attr.isAffectingSubtree) {
note.invalidateSubtreeCaches();
}
}
delete attributes[entity.attributeId];
delete attributes[attributeId];
}
else if (attributeId in attributes) {
const attr = attributes[attributeId];
// attr name cannot change
// attr name and isInheritable are immutable
attr.value = entity.value;
attr.isInheritable = entity.isInheritable;
if (attr.isAffectingSubtree) {
note.invalidateSubtreeFulltext();
}
else {
attributes[attributeId] = new Attribute(entity);
const note = notes[noteId];
note.fulltextCache = null;
}
}
else {
const attr = new Attribute(entity);
attributes[attributeId] = attr;
if (note) {
note.ownedAttributes.push(attributes[attributeId]);
note.ownedAttributes.push(attr);
if (attr.isAffectingSubtree) {
note.invalidateSubtreeCaches();
}
else {
this.invalidateThisCache();
}
}
}
}