Tools/Notes
1
0
Fork 0
mirror of https://github.com/TriliumNext/Notes.git synced 2025-11-04 15:11:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

WIP blob

Browse Source
This commit is contained in:
zadam 2023-03-15 22:44:08 +01:00
parent 1faf8225c7
commit 5a8e216dec
16 changed files with 275 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
db/migrations/0214__content_structure.sql
View File

@ -0,0 +1,10 @@
CREATE TABLE IF NOT EXISTS "blobs" (
`blobId` TEXT NOT NULL,
`content` TEXT NULL DEFAULT NULL,
`dateModified` TEXT NOT NULL,
`utcDateModified` TEXT NOT NULL,
PRIMARY KEY(`blobId`)
);
ALTER TABLE notes ADD blobId TEXT DEFAULT NULL;
ALTER TABLE note_revisions ADD blobId TEXT DEFAULT NULL;

63
db/migrations/0215__move_content_into_blobs.js Normal file
View File

@ -0,0 +1,63 @@
const sql = require("../../src/services/sql.js");
module.exports = () => {
const sql = require("../../src/services/sql");
const utils = require("../../src/services/utils");
const existingBlobIds = new Set();
for (const noteId of sql.getColumn(`SELECT noteId FROM note_contents`)) {
const row = sql.getRow(`SELECT noteId, content, dateModified, utcDateModified FROM note_contents WHERE noteId = ?`, [noteId]);
const blobId = utils.hashedBlobId(row.content);
if (!existingBlobIds.has(blobId)) {
existingBlobIds.add(blobId);
sql.insert('blobs', {
blobId,
content: row.content,
dateModified: row.dateModified,
utcDateModified: row.utcDateModified
});
sql.execute("UPDATE entity_changes SET entityName = 'blobs', entityId = ? WHERE entityName = 'note_contents' AND entityId = ?", [blobId, row.noteId]);
} else {
// duplicates
sql.execute("DELETE FROM entity_changes WHERE entityName = 'note_contents' AND entityId = ?", [row.noteId]);
}
sql.execute('UPDATE notes SET blobId = ? WHERE noteId = ?', [blobId, row.noteId]);
}
for (const noteRevisionId of sql.getColumn(`SELECT noteRevisionId FROM note_revision_contents`)) {
const row = sql.getRow(`SELECT noteRevisionId, content, utcDateModified FROM note_revision_contents WHERE noteRevisionId = ?`, [noteRevisionId]);
const blobId = utils.hashedBlobId(row.content);
if (!existingBlobIds.has(blobId)) {
existingBlobIds.add(blobId);
sql.insert('blobs', {
blobId,
content: row.content,
dateModified: row.utcDateModified,
utcDateModified: row.utcDateModified
});
sql.execute("UPDATE entity_changes SET entityName = 'blobs', entityId = ? WHERE entityName = 'note_revision_contents' AND entityId = ?", [blobId, row.noteRevisionId]);
} else {
// duplicates
sql.execute("DELETE FROM entity_changes WHERE entityName = 'note_revision_contents' AND entityId = ?", [row.noteId]);
}
sql.execute('UPDATE note_revisions SET blobId = ? WHERE noteRevisionId = ?', [blobId, row.noteRevisionId]);
}
const notesWithoutBlobIds = sql.getColumn("SELECT noteId FROM notes WHERE blobId IS NULL");
if (notesWithoutBlobIds.length > 0) {
throw new Error("BlobIds were not filled correctly in notes: " + JSON.stringify(notesWithoutBlobIds));
}
const noteRevisionsWithoutBlobIds = sql.getColumn("SELECT noteRevisionId FROM note_revisions WHERE blobId IS NULL");
if (noteRevisionsWithoutBlobIds.length > 0) {
throw new Error("BlobIds were not filled correctly in note revisions: " + JSON.stringify(noteRevisionsWithoutBlobIds));
}
};

2
db/migrations/0216__drop_content_tables.sql Normal file
View File

@ -0,0 +1,2 @@
DROP TABLE note_contents;
DROP TABLE note_revision_contents;

5
db/migrations/0215__note_ancillaries.sql → db/migrations/0217__note_ancillaries.sql
View File

@ -6,14 +6,11 @@ CREATE TABLE IF NOT EXISTS "note_ancillaries"
mime TEXT not null,
isProtected INT not null DEFAULT 0,
contentCheckSum TEXT not null,
blobId TEXT not null,
utcDateModified TEXT not null,
isDeleted INT not null,
`deleteId` TEXT DEFAULT NULL);
CREATE TABLE IF NOT EXISTS "note_ancillary_contents" (`noteAncillaryId` TEXT NOT NULL PRIMARY KEY,
`content` TEXT DEFAULT NULL,
`utcDateModified` TEXT NOT NULL);
CREATE INDEX IDX_note_ancillaries_name
on note_ancillaries (name);
CREATE UNIQUE INDEX IDX_note_ancillaries_noteId_name

2
src/becca/becca_loader.js
View File

@ -30,7 +30,7 @@ function load() {
// using raw query and passing arrays to avoid allocating new objects
// this is worth it for becca load since it happens every run and blocks the app until finished
for (const row of sql.getRawRows(`SELECT noteId, title, type, mime, isProtected, dateCreated, dateModified, utcDateCreated, utcDateModified FROM notes WHERE isDeleted = 0`)) {
for (const row of sql.getRawRows(`SELECT noteId, title, type, mime, isProtected, blobId, dateCreated, dateModified, utcDateCreated, utcDateModified FROM notes WHERE isDeleted = 0`)) {
new BNote().update(row).init();
}

66
src/becca/entities/bnote.js
View File

@ -46,6 +46,7 @@ class BNote extends AbstractBeccaEntity {
row.type,
row.mime,
row.isProtected,
row.blobId,
row.dateCreated,
row.dateModified,
row.utcDateCreated,
@ -53,19 +54,21 @@ class BNote extends AbstractBeccaEntity {
]);
}
update([noteId, title, type, mime, isProtected, dateCreated, dateModified, utcDateCreated, utcDateModified]) {
update([noteId, title, type, mime, isProtected, blobId, dateCreated, dateModified, utcDateCreated, utcDateModified]) {
// ------ Database persisted attributes ------
/** @type {string} */
this.noteId = noteId;
/** @type {string} */
this.title = title;
/** @type {boolean} */
this.isProtected = !!isProtected;
/** @type {string} */
this.type = type;
/** @type {string} */
this.mime = mime;
/** @type {boolean} */
this.isProtected = !!isProtected;
/** @type {string} */
this.blobId = blobId;
/** @type {string} */
this.dateCreated = dateCreated || dateUtils.localNowDateTime();
/** @type {string} */
@ -206,14 +209,14 @@ class BNote extends AbstractBeccaEntity {
/** @returns {*} */
getContent(silentNotFoundError = false) {
const row = sql.getRow(`SELECT content FROM note_contents WHERE noteId = ?`, [this.noteId]);
const row = sql.getRow(`SELECT content FROM blobs WHERE blobId = ?`, [this.blobId]);
if (!row) {
if (silentNotFoundError) {
return undefined;
}
else {
throw new Error(`Cannot find note content for noteId=${this.noteId}`);
throw new Error(`Cannot find note content for noteId '${this.noteId}', blobId '${this.blobId}'.`);
}
}
@ -245,8 +248,8 @@ class BNote extends AbstractBeccaEntity {
LENGTH(content) AS contentLength,
dateModified,
utcDateModified
FROM note_contents
WHERE noteId = ?`, [this.noteId]);
FROM blobs
WHERE blobId = ?`, [this.blobId]);
}
get dateCreatedObj() {
@ -276,6 +279,10 @@ class BNote extends AbstractBeccaEntity {
return JSON.parse(content);
}
isHot() {
return ['text', 'code', 'relationMap', 'canvas', 'mermaid'].includes(this.type);
}
setContent(content, ignoreMissingProtectedSession = false) {
if (content === null || content === undefined) {
throw new Error(`Cannot set null content to note '${this.noteId}'`);
@ -288,29 +295,41 @@ class BNote extends AbstractBeccaEntity {
content = Buffer.isBuffer(content) ? content : Buffer.from(content);
}
const pojo = {
noteId: this.noteId,
content: content,
dateModified: dateUtils.localNowDateTime(),
utcDateModified: dateUtils.utcNowDateTime()
};
if (this.isProtected) {
if (protectedSessionService.isProtectedSessionAvailable()) {
pojo.content = protectedSessionService.encrypt(pojo.content);
content = protectedSessionService.encrypt(content);
}
else if (!ignoreMissingProtectedSession) {
throw new Error(`Cannot update content of noteId '${this.noteId}' since we're out of protected session.`);
}
}
sql.upsert("note_contents", "noteId", pojo);
let newBlobId;
let blobNeedsInsert;
const hash = utils.hash(`${this.noteId}|${pojo.content.toString()}`);
if (this.isHot()) {
newBlobId = this.blobId || utils.randomBlobId();
blobNeedsInsert = true;
} else {
newBlobId = utils.hashedBlobId(content);
blobNeedsInsert = !sql.getValue('SELECT 1 FROM blobs WHERE blobId = ?', [newBlobId]);
}
if (blobNeedsInsert) {
const pojo = {
blobId: this.blobId,
content: content,
dateModified: dateUtils.localNowDateTime(),
utcDateModified: dateUtils.utcNowDateTime()
};
sql.upsert("blobs", "blobId", pojo);
const hash = utils.hash(`${this.blobId}|${pojo.content.toString()}`);
entityChangesService.addEntityChange({
entityName: 'note_contents',
entityId: this.noteId,
entityName: 'blobs',
entityId: this.blobId,
hash: hash,
isErased: false,
utcDateChanged: pojo.utcDateModified,
@ -318,11 +337,17 @@ class BNote extends AbstractBeccaEntity {
});
eventService.emit(eventService.ENTITY_CHANGED, {
entityName: 'note_contents',
entityName: 'blobs',
entity: this
});
}
if (newBlobId !== this.blobId) {
this.blobId = newBlobId;
this.save();
}
}
setJsonContent(content) {
this.setContent(JSON.stringify(content, null, '\t'));
}
@ -1517,6 +1542,7 @@ class BNote extends AbstractBeccaEntity {
isProtected: this.isProtected,
type: this.type,
mime: this.mime,
blobId: this.blobId,
isDeleted: false,
dateCreated: this.dateCreated,
dateModified: this.dateModified,

49
src/becca/entities/bnote_revision.js
View File

@ -35,6 +35,8 @@ class BNoteRevision extends AbstractBeccaEntity {
/** @type {string} */
this.title = row.title;
/** @type {string} */
this.blobId = row.blobId;
/** @type {string} */
this.dateLastEdited = row.dateLastEdited;
/** @type {string} */
this.dateCreated = row.dateCreated;
@ -74,14 +76,14 @@ class BNoteRevision extends AbstractBeccaEntity {
/** @returns {*} */
getContent(silentNotFoundError = false) {
const res = sql.getRow(`SELECT content FROM note_revision_contents WHERE noteRevisionId = ?`, [this.noteRevisionId]);
const res = sql.getRow(`SELECT content FROM blobs WHERE blobId = ?`, [this.blobId]);
if (!res) {
if (silentNotFoundError) {
return undefined;
}
else {
throw new Error(`Cannot find note revision content for noteRevisionId=${this.noteRevisionId}`);
throw new Error(`Cannot find note revision content for noteRevisionId '${this.noteRevisionId}', blobId '${this.blobId}'`);
}
}
@ -107,28 +109,34 @@ class BNoteRevision extends AbstractBeccaEntity {
}
setContent(content) {
if (this.isProtected) {
if (protectedSessionService.isProtectedSessionAvailable()) {
content = protectedSessionService.encrypt(content);
}
else {
throw new Error(`Cannot update content of noteRevisionId '${this.noteRevisionId}' since we're out of protected session.`);
}
}
this.blobId = utils.hashedBlobId(content);
const blobAlreadyExists = !sql.getValue('SELECT 1 FROM blobs WHERE blobId = ?', [this.blobId]);
if (!blobAlreadyExists) {
const pojo = {
noteRevisionId: this.noteRevisionId,
blobId: this.blobId,
content: content,
dateModified: dateUtils.localNowDate(),
utcDateModified: dateUtils.utcNowDateTime()
};
if (this.isProtected) {
if (protectedSessionService.isProtectedSessionAvailable()) {
pojo.content = protectedSessionService.encrypt(pojo.content);
}
else {
throw new Error(`Cannot update content of noteRevisionId=${this.noteRevisionId} since we're out of protected session.`);
}
}
sql.upsert("note_revision_contents", "noteRevisionId", pojo);
sql.insert("blobs", pojo);
const hash = utils.hash(`${this.noteRevisionId}|${pojo.content.toString()}`);
entityChangesService.addEntityChange({
entityName: 'note_revision_contents',
entityId: this.noteRevisionId,
entityName: 'blobs',
entityId: this.blobId,
hash: hash,
isErased: false,
utcDateChanged: this.getUtcDateChanged(),
@ -136,15 +144,7 @@ class BNoteRevision extends AbstractBeccaEntity {
});
}
/** @returns {{contentLength, dateModified, utcDateModified}} */
getContentMetadata() {
return sql.getRow(`
SELECT
LENGTH(content) AS contentLength,
dateModified,
utcDateModified
FROM note_revision_contents
WHERE noteRevisionId = ?`, [this.noteRevisionId]);
this.save(); // saving this.blobId
}
beforeSaving() {
@ -161,6 +161,7 @@ class BNoteRevision extends AbstractBeccaEntity {
mime: this.mime,
isProtected: this.isProtected,
title: this.title,
blobId: this.blobId,
dateLastEdited: this.dateLastEdited,
dateCreated: this.dateCreated,
utcDateLastEdited: this.utcDateLastEdited,

2
src/public/app/entities/fnote_complement.js
View File

@ -27,7 +27,7 @@ class FNoteComplement {
/** @type {string} */
this.utcDateModified = row.utcDateModified;
// "combined" date modified give larger out of note's and note_content's dateModified
// "combined" date modified give larger out of note's and blob's dateModified
/** @type {string} */
this.combinedDateModified = row.combinedDateModified;

12
src/public/app/services/froca.js
View File

@ -35,7 +35,7 @@ class Froca {
this.attributes = {};
/** @type {Object.<string, Promise<FNoteComplement>>} */
this.noteComplementPromises = {};
this.blobPromises = {};
this.addResp(resp);
}
@ -314,20 +314,20 @@ class Froca {
* @returns {Promise<FNoteComplement>}
*/
async getNoteComplement(noteId) {
if (!this.noteComplementPromises[noteId]) {
this.noteComplementPromises[noteId] = server.get(`notes/${noteId}`)
if (!this.blobPromises[noteId]) {
this.blobPromises[noteId] = server.get(`notes/${noteId}`)
.then(row => new FNoteComplement(row))
.catch(e => console.error(`Cannot get note complement for note '${noteId}'`));
// we don't want to keep large payloads forever in memory, so we clean that up quite quickly
// this cache is more meant to share the data between different components within one business transaction (e.g. loading of the note into the tab context and all the components)
// this is also a workaround for missing invalidation after change
this.noteComplementPromises[noteId].then(
() => setTimeout(() => this.noteComplementPromises[noteId] = null, 1000)
this.blobPromises[noteId].then(
() => setTimeout(() => this.blobPromises[noteId] = null, 1000)
);
}
return await this.noteComplementPromises[noteId];
return await this.blobPromises[noteId];
}
}

6
src/public/app/services/froca_updater.js
View File

@ -19,10 +19,10 @@ async function processEntityChanges(entityChanges) {
processAttributeChange(loadResults, ec);
} else if (ec.entityName === 'note_reordering') {
processNoteReordering(loadResults, ec);
} else if (ec.entityName === 'note_contents') {
delete froca.noteComplementPromises[ec.entityId];
} else if (ec.entityName === 'blobs') {
delete froca.blobPromises[ec.entityId];
loadResults.addNoteContent(ec.entityId, ec.componentId);
loadResults.addNoteContent(ec.noteIds, ec.componentId);
} else if (ec.entityName === 'note_revisions') {
loadResults.addNoteRevision(ec.entityId, ec.noteId, ec.componentId);
} else if (ec.entityName === 'note_revision_contents') {

4
src/public/app/services/load_results.js
View File

@ -94,9 +94,11 @@ export default class LoadResults {
return componentIds && componentIds.find(sId => sId !== componentId) !== undefined;
}
addNoteContent(noteId, componentId) {
addNoteContent(noteIds, componentId) {
for (const noteId of noteIds) {
this.contentNoteIdToComponentId.push({noteId, componentId});
}
}
isNoteContentReloaded(noteId, componentId) {
if (!noteId) {

2
src/services/app_info.js
View File

@ -4,7 +4,7 @@ const build = require('./build');
const packageJson = require('../../package');
const {TRILIUM_DATA_DIR} = require('./data_dir');
const APP_DB_VERSION = 215;
const APP_DB_VERSION = 217;
const SYNC_VERSION = 30;
const CLIPPER_PROTOCOL_VERSION = "1.0";

154
src/services/consistency_checks.js
View File

@ -383,86 +383,86 @@ class ConsistencyChecks {
}
});
this.findAndFixIssues(`
SELECT notes.noteId, notes.isProtected, notes.type, notes.mime
FROM notes
LEFT JOIN note_contents USING (noteId)
WHERE note_contents.noteId IS NULL`,
({noteId, isProtected, type, mime}) => {
if (this.autoFix) {
// it might be possible that the note_content is not available only because of the interrupted
// sync, and it will come later. It's therefore important to guarantee that this artifical
// record won't overwrite the real one coming from the sync.
const fakeDate = "2000-01-01 00:00:00Z";
// manually creating row since this can also affect deleted notes
sql.upsert("note_contents", "noteId", {
noteId: noteId,
content: getBlankContent(isProtected, type, mime),
utcDateModified: fakeDate,
dateModified: fakeDate
});
const hash = utils.hash(utils.randomString(10));
entityChangesService.addEntityChange({
entityName: 'note_contents',
entityId: noteId,
hash: hash,
isErased: false,
utcDateChanged: fakeDate,
isSynced: true
});
this.reloadNeeded = true;
logFix(`Note '${noteId}' content was set to empty string since there was no corresponding row`);
} else {
logError(`Note '${noteId}' content row does not exist`);
}
});
// this.findAndFixIssues(`
// SELECT notes.noteId, notes.isProtected, notes.type, notes.mime
// FROM notes
// LEFT JOIN note_contents USING (noteId)
// WHERE note_contents.noteId IS NULL`,
// ({noteId, isProtected, type, mime}) => {
// if (this.autoFix) {
// // it might be possible that the note_content is not available only because of the interrupted
// // sync, and it will come later. It's therefore important to guarantee that this artifical
// // record won't overwrite the real one coming from the sync.
// const fakeDate = "2000-01-01 00:00:00Z";
//
// // manually creating row since this can also affect deleted notes
// sql.upsert("note_contents", "noteId", {
// noteId: noteId,
// content: getBlankContent(isProtected, type, mime),
// utcDateModified: fakeDate,
// dateModified: fakeDate
// });
//
// const hash = utils.hash(utils.randomString(10));
//
// entityChangesService.addEntityChange({
// entityName: 'note_contents',
// entityId: noteId,
// hash: hash,
// isErased: false,
// utcDateChanged: fakeDate,
// isSynced: true
// });
//
// this.reloadNeeded = true;
//
// logFix(`Note '${noteId}' content was set to empty string since there was no corresponding row`);
// } else {
// logError(`Note '${noteId}' content row does not exist`);
// }
// });
if (sqlInit.getDbSize() < 500000) {
// querying for "content IS NULL" is expensive since content is not indexed. See e.g. https://github.com/zadam/trilium/issues/2887
this.findAndFixIssues(`
SELECT notes.noteId, notes.type, notes.mime
FROM notes
JOIN note_contents USING (noteId)
WHERE isDeleted = 0
AND isProtected = 0
AND content IS NULL`,
({noteId, type, mime}) => {
if (this.autoFix) {
const note = becca.getNote(noteId);
const blankContent = getBlankContent(false, type, mime);
note.setContent(blankContent);
this.reloadNeeded = true;
logFix(`Note '${noteId}' content was set to '${blankContent}' since it was null even though it is not deleted`);
} else {
logError(`Note '${noteId}' content is null even though it is not deleted`);
}
});
// this.findAndFixIssues(`
// SELECT notes.noteId, notes.type, notes.mime
// FROM notes
// JOIN note_contents USING (noteId)
// WHERE isDeleted = 0
// AND isProtected = 0
// AND content IS NULL`,
// ({noteId, type, mime}) => {
// if (this.autoFix) {
// const note = becca.getNote(noteId);
// const blankContent = getBlankContent(false, type, mime);
// note.setContent(blankContent);
//
// this.reloadNeeded = true;
//
// logFix(`Note '${noteId}' content was set to '${blankContent}' since it was null even though it is not deleted`);
// } else {
// logError(`Note '${noteId}' content is null even though it is not deleted`);
// }
// });
}
this.findAndFixIssues(`
SELECT note_revisions.noteRevisionId
FROM note_revisions
LEFT JOIN note_revision_contents USING (noteRevisionId)
WHERE note_revision_contents.noteRevisionId IS NULL`,
({noteRevisionId}) => {
if (this.autoFix) {
noteRevisionService.eraseNoteRevisions([noteRevisionId]);
this.reloadNeeded = true;
logFix(`Note revision content '${noteRevisionId}' was set to erased since it did not exist.`);
} else {
logError(`Note revision content '${noteRevisionId}' does not exist`);
}
});
// this.findAndFixIssues(`
// SELECT note_revisions.noteRevisionId
// FROM note_revisions
// LEFT JOIN note_revision_contents USING (noteRevisionId)
// WHERE note_revision_contents.noteRevisionId IS NULL`,
// ({noteRevisionId}) => {
// if (this.autoFix) {
// noteRevisionService.eraseNoteRevisions([noteRevisionId]);
//
// this.reloadNeeded = true;
//
// logFix(`Note revision content '${noteRevisionId}' was set to erased since it did not exist.`);
// } else {
// logError(`Note revision content '${noteRevisionId}' does not exist`);
// }
// });
this.findAndFixIssues(`
SELECT parentNoteId
@ -656,11 +656,11 @@ class ConsistencyChecks {
findEntityChangeIssues() {
this.runEntityChangeChecks("notes", "noteId");
this.runEntityChangeChecks("note_contents", "noteId");
//this.runEntityChangeChecks("note_contents", "noteId");
this.runEntityChangeChecks("note_revisions", "noteRevisionId");
this.runEntityChangeChecks("note_revision_contents", "noteRevisionId");
//this.runEntityChangeChecks("note_revision_contents", "noteRevisionId");
this.runEntityChangeChecks("note_ancillaries", "noteAncillaryId");
this.runEntityChangeChecks("note_ancillary_contents", "noteAncillaryId");
//this.runEntityChangeChecks("note_ancillary_contents", "noteAncillaryId");
this.runEntityChangeChecks("branches", "branchId");
this.runEntityChangeChecks("attributes", "attributeId");
this.runEntityChangeChecks("etapi_tokens", "etapiTokenId");

17
src/services/utils.js
View File

@ -25,6 +25,19 @@ function md5(content) {
return crypto.createHash('md5').update(content).digest('hex');
}
function hashedBlobId(content) {
// sha512 is faster than sha256
const base64Hash = crypto.createHash('sha512').update(content).digest('base64');
// 20 characters of base64 gives us 120 bit of entropy which is plenty enough
return base64Hash.substr(0, 20);
}
function randomBlobId(content) {
// underscore prefix to easily differentiate the random as opposed to hashed
return '_' + randomString(19);
}
function toBase64(plainText) {
return Buffer.from(plainText).toString('base64');
}
@ -343,5 +356,7 @@ module.exports = {
deferred,
removeDiacritic,
normalize,
filterAttributeName
filterAttributeName,
hashedBlobId,
randomBlobId
};

5
src/services/ws.js
View File

@ -129,13 +129,14 @@ function fillInAdditionalProperties(entityChange) {
entityChange.positions[childBranch.branchId] = childBranch.notePosition;
}
}
}
else if (entityChange.entityName === 'options') {
} else if (entityChange.entityName === 'options') {
entityChange.entity = becca.getOption(entityChange.entityId);
if (!entityChange.entity) {
entityChange.entity = sql.getRow(`SELECT * FROM options WHERE name = ?`, [entityChange.entityId]);
}
} else if (entityChange.entityName === 'blob') {
entityChange.noteIds = sql.getColumn("SELECT noteId FROM notes WHERE blobId = ? AND isDeleted = 0", [entityChange.entityId]);
}
if (entityChange.entity instanceof AbstractBeccaEntity) {