diff --git a/src/becca/entities/abstract_becca_entity.js b/src/becca/entities/abstract_becca_entity.js index 4806063b5..bf4c4597d 100644 --- a/src/becca/entities/abstract_becca_entity.js +++ b/src/becca/entities/abstract_becca_entity.js @@ -252,7 +252,7 @@ class AbstractBeccaEntity { if (this.hasStringContent()) { return content === null ? "" - : content.toString("UTF-8"); + : content.toString("utf-8"); } else { return content; } diff --git a/src/becca/entities/bnote.js b/src/becca/entities/bnote.js index a02ca99aa..af68ca8d6 100644 --- a/src/becca/entities/bnote.js +++ b/src/becca/entities/bnote.js @@ -1120,7 +1120,8 @@ class BNote extends AbstractBeccaEntity { SELECT attachments.* FROM attachments WHERE parentId = ? - AND isDeleted = 0`, [this.noteId]) + AND isDeleted = 0 + ORDER BY position`, [this.noteId]) .map(row => new BAttachment(row)); } @@ -1142,7 +1143,8 @@ class BNote extends AbstractBeccaEntity { FROM attachments WHERE parentId = ? AND role = ? - AND isDeleted = 0`, [this.noteId, role]) + AND isDeleted = 0 + ORDER BY position`, [this.noteId, role]) .map(row => new BAttachment(row)); } diff --git a/src/services/data_encryption.js b/src/services/data_encryption.js index 12398ffc7..df513868b 100644 --- a/src/services/data_encryption.js +++ b/src/services/data_encryption.js @@ -112,7 +112,7 @@ function decryptString(dataKey, cipherText) { throw new Error("Could not decrypt string."); } - return buffer.toString('UTF-8'); + return buffer.toString('utf-8'); } module.exports = { diff --git a/src/services/etapi_tokens.js b/src/services/etapi_tokens.js index 23e71e4f0..dcf3a178f 100644 --- a/src/services/etapi_tokens.js +++ b/src/services/etapi_tokens.js @@ -34,7 +34,7 @@ function parseAuthToken(auth) { // allow also basic auth format for systems which allow this type of authentication // expect ETAPI token in the password field, require "etapi" username // https://github.com/zadam/trilium/issues/3181 - const basicAuthStr = utils.fromBase64(auth.substring(6)).toString("UTF-8"); + const basicAuthStr = utils.fromBase64(auth.substring(6)).toString("utf-8"); const basicAuthChunks = basicAuthStr.split(":"); if (basicAuthChunks.length !== 2) { diff --git a/src/services/export/zip.js b/src/services/export/zip.js index f56a7c741..88da1c8b8 100644 --- a/src/services/export/zip.js +++ b/src/services/export/zip.js @@ -122,7 +122,7 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true) * @param {Object.} existingFileNames * @returns {NoteMeta|null} */ - function getNoteMeta(branch, parentMeta, existingFileNames) { + function createNoteMeta(branch, parentMeta, existingFileNames) { const note = branch.getNote(); if (note.hasOwnedLabel('excludeFromExport')) { @@ -200,6 +200,7 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true) attMeta.title = attachment.title; attMeta.role = attachment.role; attMeta.mime = attachment.mime; + attMeta.position = attachment.position; attMeta.dataFileName = getDataFileName( null, attachment.mime, @@ -217,7 +218,7 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true) const childExistingNames = {}; for (const childBranch of childBranches) { - const note = getNoteMeta(childBranch, meta, childExistingNames); + const note = createNoteMeta(childBranch, meta, childExistingNames); // can be undefined if export is disabled for this note if (note) { @@ -234,7 +235,7 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true) * @param {NoteMeta} sourceMeta * @return {string|null} */ - function getTargetUrl(targetNoteId, sourceMeta) { + function getNoteTargetUrl(targetNoteId, sourceMeta) { const targetMeta = noteIdToMeta[targetNoteId]; if (!targetMeta) { @@ -271,15 +272,29 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true) * @param {NoteMeta} noteMeta * @return {string} */ - function findLinks(content, noteMeta) { + function rewriteLinks(content, noteMeta) { content = content.replace(/src="[^"]*api\/images\/([a-zA-Z0-9_]+)\/[^"]*"/g, (match, targetNoteId) => { - const url = getTargetUrl(targetNoteId, noteMeta); + const url = getNoteTargetUrl(targetNoteId, noteMeta); + + return url ? `src="${url}"` : match; + }); + + content = content.replace(/src="[^"]*api\/attachments\/([a-zA-Z0-9_]+)\/image\/[^"]*"/g, (match, targetAttachmentId) => { + let url; + + const attachmentMeta = noteMeta.attachments.find(attMeta => attMeta.attachmentId === targetAttachmentId); + if (attachmentMeta) { + // easy job here, because attachment will be in the same directory as the note's data file. + url = attachmentMeta.dataFileName; + } else { + log.info(`Could not find attachment meta object for attachmentId '${targetAttachmentId}'`); + } return url ? `src="${url}"` : match; }); content = content.replace(/href="[^"]*#root[a-zA-Z0-9_\/]*\/([a-zA-Z0-9_]+)\/?"/g, (match, targetNoteId) => { - const url = getTargetUrl(targetNoteId, noteMeta); + const url = getNoteTargetUrl(targetNoteId, noteMeta); return url ? `href="${url}"` : match; }); @@ -297,7 +312,7 @@ async function exportToZip(taskContext, branch, format, res, setHeaders = true) if (['html', 'markdown'].includes(noteMeta.format)) { content = content.toString(); - content = findLinks(content, noteMeta); + content = rewriteLinks(content, noteMeta); } if (noteMeta.format === 'html') { @@ -347,7 +362,7 @@ ${markdownContent}`; log.info(`Exporting note '${noteMeta.noteId}'`); if (noteMeta.isClone) { - const targetUrl = getTargetUrl(noteMeta.noteId, noteMeta); + const targetUrl = getNoteTargetUrl(noteMeta.noteId, noteMeta); let content = `

This is a clone of a note. Go to its primary location.

`; @@ -404,7 +419,7 @@ ${markdownContent}`; const escapedTitle = utils.escapeHtml(`${meta.prefix ? `${meta.prefix} - ` : ''}${meta.title}`); if (meta.dataFileName) { - const targetUrl = getTargetUrl(meta.noteId, rootMeta); + const targetUrl = getNoteTargetUrl(meta.noteId, rootMeta); html += `${escapedTitle}`; } @@ -449,7 +464,7 @@ ${markdownContent}`; while (!firstNonEmptyNote) { if (curMeta.dataFileName) { - firstNonEmptyNote = getTargetUrl(curMeta.noteId, rootMeta); + firstNonEmptyNote = getNoteTargetUrl(curMeta.noteId, rootMeta); } if (curMeta.children && curMeta.children.length > 0) { @@ -486,7 +501,7 @@ ${markdownContent}`; } const existingFileNames = format === 'html' ? ['navigation', 'index'] : []; - const rootMeta = getNoteMeta(branch, { notePath: [] }, existingFileNames); + const rootMeta = createNoteMeta(branch, { notePath: [] }, existingFileNames); const metaFile = { formatVersion: 1, diff --git a/src/services/import/single.js b/src/services/import/single.js index cf7496c2c..fbc8ca5a7 100644 --- a/src/services/import/single.js +++ b/src/services/import/single.js @@ -61,7 +61,7 @@ function importFile(taskContext, file, parentNote) { function importCodeNote(taskContext, file, parentNote) { const title = utils.getNoteTitle(file.originalname, taskContext.data.replaceUnderscoresWithSpaces); - const content = file.buffer.toString("UTF-8"); + const content = file.buffer.toString("utf-8"); const detectedMime = mimeService.getMime(file.originalname) || file.mimetype; const mime = mimeService.normalizeMimeType(detectedMime); @@ -81,7 +81,7 @@ function importCodeNote(taskContext, file, parentNote) { function importPlainText(taskContext, file, parentNote) { const title = utils.getNoteTitle(file.originalname, taskContext.data.replaceUnderscoresWithSpaces); - const plainTextContent = file.buffer.toString("UTF-8"); + const plainTextContent = file.buffer.toString("utf-8"); const htmlContent = convertTextToHtml(plainTextContent); const {note} = noteService.createNewNote({ @@ -119,7 +119,7 @@ function convertTextToHtml(text) { function importMarkdown(taskContext, file, parentNote) { const title = utils.getNoteTitle(file.originalname, taskContext.data.replaceUnderscoresWithSpaces); - const markdownContent = file.buffer.toString("UTF-8"); + const markdownContent = file.buffer.toString("utf-8"); const reader = new commonmark.Parser(); const writer = new commonmark.HtmlRenderer(); @@ -158,7 +158,7 @@ function handleH1(content, title) { function importHtml(taskContext, file, parentNote) { const title = utils.getNoteTitle(file.originalname, taskContext.data.replaceUnderscoresWithSpaces); - let content = file.buffer.toString("UTF-8"); + let content = file.buffer.toString("utf-8"); content = htmlSanitizer.sanitize(content); diff --git a/src/services/import/zip.js b/src/services/import/zip.js index 05e75b46c..301f68163 100644 --- a/src/services/import/zip.js +++ b/src/services/import/zip.js @@ -25,9 +25,11 @@ const BAttachment = require("../../becca/entities/battachment"); async function importZip(taskContext, fileBuffer, importRootNote) { /** @type {Object.} maps from original noteId (in ZIP file) to newly generated noteId */ const noteIdMap = {}; + /** @type {Object.} maps from original attachmentId (in ZIP file) to newly generated attachmentId */ + const attachmentIdMap = {}; const attributes = []; // path => noteId, used only when meta file is not available - /** @type {Object.} path => noteId */ + /** @type {Object.} path => noteId | attachmentId */ const createdPaths = { '/': importRootNote.noteId, '\\': importRootNote.noteId }; const mdReader = new commonmark.Parser(); const mdWriter = new commonmark.HtmlRenderer(); @@ -38,9 +40,9 @@ async function importZip(taskContext, fileBuffer, importRootNote) { const createdNoteIds = new Set(); function getNewNoteId(origNoteId) { - // in case the original noteId is empty. This probably shouldn't happen, but still good to have this precaution if (!origNoteId.trim()) { - return ""; + // this probably shouldn't happen, but still good to have this precaution + return "empty_note_id"; } if (origNoteId === 'root' || origNoteId.startsWith("_")) { @@ -55,7 +57,40 @@ async function importZip(taskContext, fileBuffer, importRootNote) { return noteIdMap[origNoteId]; } - /** @returns {{noteMeta: NoteMeta, parentNoteMeta: NoteMeta, attachmentMeta: AttachmentMeta}} */ + function getNewAttachmentId(origAttachmentId) { + if (!origAttachmentId.trim()) { + // this probably shouldn't happen, but still good to have this precaution + return "empty_attachment_id"; + } + + if (!attachmentIdMap[origAttachmentId]) { + attachmentIdMap[origAttachmentId] = utils.newEntityId(); + } + + return attachmentIdMap[origAttachmentId]; + } + + /** + * @param {NoteMeta} parentNoteMeta + * @param {string} dataFileName + */ + function getAttachmentMeta(parentNoteMeta, dataFileName) { + for (const noteMeta of parentNoteMeta.children) { + for (const attachmentMeta of noteMeta.attachments || []) { + if (attachmentMeta.dataFileName === dataFileName) { + return { + parentNoteMeta, + noteMeta, + attachmentMeta + }; + } + } + } + + return {}; + } + + /** @returns {{noteMeta: NoteMeta|undefined, parentNoteMeta: NoteMeta|undefined, attachmentMeta: AttachmentMeta|undefined}} */ function getMeta(filePath) { if (!metaFile) { return {}; @@ -63,16 +98,17 @@ async function importZip(taskContext, fileBuffer, importRootNote) { const pathSegments = filePath.split(/[\/\\]/g); + /** @type {NoteMeta} */ let cursor = { isImportRoot: true, children: metaFile.files }; + /** @type {NoteMeta} */ let parent; - let attachmentMeta = false; for (const segment of pathSegments) { - if (!cursor || !cursor.children || cursor.children.length === 0) { + if (!cursor?.children?.length) { return {}; } @@ -80,26 +116,13 @@ async function importZip(taskContext, fileBuffer, importRootNote) { cursor = parent.children.find(file => file.dataFileName === segment || file.dirFileName === segment); if (!cursor) { - for (const file of parent.children) { - for (const attachment of file.attachments || []) { - if (attachment.dataFileName === segment) { - cursor = file; - attachmentMeta = attachment; - break; - } - } - - if (cursor) { - break; - } - } + return getAttachmentMeta(parent, segment); } } return { parentNoteMeta: parent, - noteMeta: cursor, - attachmentMeta + noteMeta: cursor }; } @@ -119,12 +142,10 @@ async function importZip(taskContext, fileBuffer, importRootNote) { if (parentPath === '.') { parentNoteId = importRootNote.noteId; - } - else if (parentPath in createdPaths) { + } else if (parentPath in createdPaths) { parentNoteId = createdPaths[parentPath]; - } - else { - // ZIP allows creating out of order records - i.e. file in a directory can appear in the ZIP stream before actual directory + } else { + // ZIP allows creating out of order records - i.e., file in a directory can appear in the ZIP stream before actual directory parentNoteId = saveDirectory(parentPath); } } @@ -142,6 +163,8 @@ async function importZip(taskContext, fileBuffer, importRootNote) { return getNewNoteId(noteMeta.noteId); } + // in case we lack metadata, we treat e.g. "Programming.html" and "Programming" as the same note + // (one data file, the other directory for children) const filePathNoExt = utils.removeTextFileExtension(filePath); if (filePathNoExt in createdPaths) { @@ -214,16 +237,15 @@ async function importZip(taskContext, fileBuffer, importRootNote) { const { parentNoteMeta, noteMeta } = getMeta(filePath); const noteId = getNoteId(noteMeta, filePath); - const noteTitle = utils.getNoteTitle(filePath, taskContext.data.replaceUnderscoresWithSpaces, noteMeta); - const parentNoteId = getParentNoteId(filePath, parentNoteMeta); - let note = becca.getNote(noteId); - - if (note) { + if (becca.getNote(noteId)) { return; } - ({note} = noteService.createNewNote({ + const noteTitle = utils.getNoteTitle(filePath, taskContext.data.replaceUnderscoresWithSpaces, noteMeta); + const parentNoteId = getParentNoteId(filePath, parentNoteMeta); + + const {note} = noteService.createNewNote({ parentNoteId: parentNoteId, title: noteTitle, content: '', @@ -234,20 +256,21 @@ async function importZip(taskContext, fileBuffer, importRootNote) { isExpanded: noteMeta ? noteMeta.isExpanded : false, notePosition: (noteMeta && firstNote) ? noteMeta.notePosition : undefined, isProtected: importRootNote.isProtected && protectedSessionService.isProtectedSessionAvailable(), - })); + }); createdNoteIds.add(note.noteId); saveAttributes(note, noteMeta); - if (!firstNote) { - firstNote = note; - } + firstNote = firstNote || note; return noteId; } - function getNoteIdFromRelativeUrl(url, filePath) { + /** + * @returns {{attachmentId: string}|{noteId: string}} + */ + function getEntityIdFromRelativeUrl(url, filePath) { while (url.startsWith("./")) { url = url.substr(2); } @@ -266,10 +289,17 @@ async function importZip(taskContext, fileBuffer, importRootNote) { absUrl += `${absUrl.length > 0 ? '/' : ''}${url}`; - const {noteMeta} = getMeta(absUrl); + const { noteMeta, attachmentMeta } = getMeta(absUrl); - const targetNoteId = getNoteId(noteMeta, absUrl); - return targetNoteId; + if (attachmentMeta) { + return { + attachmentId: getNewAttachmentId(attachmentMeta.attachmentId) + }; + } else { // don't check for noteMeta since it's not mandatory for notes + return { + noteId: getNoteId(noteMeta, absUrl) + }; + } } /** @@ -299,9 +329,9 @@ async function importZip(taskContext, fileBuffer, importRootNote) { content = content.replace(/src="([^"]*)"/g, (match, url) => { try { - url = decodeURIComponent(url); + url = decodeURIComponent(url).trim(); } catch (e) { - log.error(`Cannot parse image URL '${url}', keeping original (${e}).`); + log.error(`Cannot parse image URL '${url}', keeping original. Error: ${e.message}.`); return `src="${url}"`; } @@ -309,20 +339,22 @@ async function importZip(taskContext, fileBuffer, importRootNote) { return match; } - const targetNoteId = getNoteIdFromRelativeUrl(url, filePath); + const target = getEntityIdFromRelativeUrl(url, filePath); - if (!targetNoteId) { + if (target.noteId) { + return `src="api/images/${target.noteId}/${path.basename(url)}"`; + } else if (target.attachmentId) { + return `src="api/attachments/${target.attachmentId}/image/${path.basename(url)}"`; + } else { return match; } - - return `src="api/images/${targetNoteId}/${path.basename(url)}"`; }); content = content.replace(/href="([^"]*)"/g, (match, url) => { try { - url = decodeURIComponent(url); + url = decodeURIComponent(url).trim(); } catch (e) { - log.error(`Cannot parse link URL '${url}', keeping original (${e}).`); + log.error(`Cannot parse link URL '${url}', keeping original. Error: ${e.message}.`); return `href="${url}"`; } @@ -331,13 +363,15 @@ async function importZip(taskContext, fileBuffer, importRootNote) { return match; } - const targetNoteId = getNoteIdFromRelativeUrl(url, filePath); + const target = getEntityIdFromRelativeUrl(url, filePath); - if (!targetNoteId) { + if (!target.noteId) { return match; } - return `href="#root/${targetNoteId}"`; + // FIXME for linking attachments + + return `href="#root/${target.noteId}"`; }); content = content.replace(/data-note-path="([^"]*)"/g, (match, notePath) => { @@ -408,7 +442,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) { * @param {Buffer} content */ function saveNote(filePath, content) { - const {parentNoteMeta, noteMeta, attachmentMeta} = getMeta(filePath); + const { parentNoteMeta, noteMeta, attachmentMeta } = getMeta(filePath); if (noteMeta?.noImport) { return; @@ -418,10 +452,12 @@ async function importZip(taskContext, fileBuffer, importRootNote) { if (attachmentMeta) { const attachment = new BAttachment({ + attachmentId: getNewAttachmentId(attachmentMeta.attachmentId), parentId: noteId, title: attachmentMeta.title, role: attachmentMeta.role, - mime: attachmentMeta.mime + mime: attachmentMeta.mime, + position: attachmentMeta.position }); attachment.setContent(content); @@ -448,11 +484,11 @@ async function importZip(taskContext, fileBuffer, importRootNote) { return; } - let {type, mime} = noteMeta ? noteMeta : detectFileTypeAndMime(taskContext, filePath); + let { type, mime } = noteMeta ? noteMeta : detectFileTypeAndMime(taskContext, filePath); type = resolveNoteType(type); if (type !== 'file' && type !== 'image') { - content = content.toString("UTF-8"); + content = content.toString("utf-8"); } const noteTitle = utils.getNoteTitle(filePath, taskContext.data.replaceUnderscoresWithSpaces, noteMeta); @@ -496,7 +532,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) { mime, prefix: noteMeta ? noteMeta.prefix : '', isExpanded: noteMeta ? noteMeta.isExpanded : false, - // root notePosition should be ignored since it relates to original document + // root notePosition should be ignored since it relates to the original document // now import root should be placed after existing notes into new parent notePosition: (noteMeta && firstNote) ? noteMeta.notePosition : undefined, isProtected: isProtected, @@ -506,13 +542,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) { saveAttributes(note, noteMeta); - if (!firstNote) { - firstNote = note; - } - - if (type === 'text') { - filePath = utils.removeTextFileExtension(filePath); - } + firstNote = firstNote || note; } if (!noteMeta && (type === 'file' || type === 'image')) { @@ -533,7 +563,7 @@ async function importZip(taskContext, fileBuffer, importRootNote) { if (filePath === '!!!meta.json') { const content = await readContent(zipfile, entry); - metaFile = JSON.parse(content.toString("UTF-8")); + metaFile = JSON.parse(content.toString("utf-8")); } zipfile.readEntry(); @@ -597,6 +627,7 @@ function normalizeFilePath(filePath) { return filePath; } +/** @returns {Promise} */ function streamToBuffer(stream) { const chunks = []; stream.on('data', chunk => chunks.push(chunk)); @@ -604,7 +635,7 @@ function streamToBuffer(stream) { return new Promise((res, rej) => stream.on('end', () => res(Buffer.concat(chunks)))); } -/** @returns {Buffer} */ +/** @returns {Promise} */ function readContent(zipfile, entry) { return new Promise((res, rej) => { zipfile.openReadStream(entry, function(err, readStream) { @@ -627,8 +658,6 @@ function readZipFile(buffer, processEntryCallback) { } function resolveNoteType(type) { - type = type || 'text'; - // BC for ZIPs created in Triliun 0.57 and older if (type === 'relation-map') { type = 'relationMap'; @@ -638,7 +667,7 @@ function resolveNoteType(type) { type = 'webView'; } - return type; + return type || "text"; } diff --git a/src/services/meta/attachment_meta.js b/src/services/meta/attachment_meta.js index 1e49f611d..8baa568a1 100644 --- a/src/services/meta/attachment_meta.js +++ b/src/services/meta/attachment_meta.js @@ -7,6 +7,8 @@ class AttachmentMeta { role; /** @type {string} */ mime; + /** @type {integer} */ + position; /** @type {string} */ dataFileName; } diff --git a/src/services/meta/attribute_meta.js b/src/services/meta/attribute_meta.js index ca4cd144d..c77e557bc 100644 --- a/src/services/meta/attribute_meta.js +++ b/src/services/meta/attribute_meta.js @@ -3,7 +3,7 @@ class AttributeMeta { type; /** @type {string} */ name; - /** @type {boolean} */ + /** @type {string} */ value; /** @type {boolean} */ isInheritable; diff --git a/src/services/sync.js b/src/services/sync.js index d6b8d004a..b229eea42 100644 --- a/src/services/sync.js +++ b/src/services/sync.js @@ -323,7 +323,7 @@ function getEntityChangeRow(entityName, entityId) { if (entityName === 'blobs' && entity.content !== null) { if (typeof entity.content === 'string') { - entity.content = Buffer.from(entity.content, 'UTF-8'); + entity.content = Buffer.from(entity.content, 'utf-8'); } entity.content = entity.content.toString("base64"); diff --git a/src/share/shaca/entities/snote.js b/src/share/shaca/entities/snote.js index f28260deb..257a1dd98 100644 --- a/src/share/shaca/entities/snote.js +++ b/src/share/shaca/entities/snote.js @@ -110,7 +110,7 @@ class SNote extends AbstractShacaEntity { if (this.hasStringContent()) { return content === null ? "" - : content.toString("UTF-8"); + : content.toString("utf-8"); } else { return content;