basic enex import, closes #194

package.json

@@ -56,6 +56,7 @@
     "request-promise": "4.2.2",
     "rimraf": "2.6.2",
     "sanitize-filename": "1.6.1",
+    "sax": "^1.2.4",
     "serve-favicon": "2.5.0",
     "session-file-store": "1.2.0",
     "simple-node-logger": "0.93.40",

src/public/javascripts/services/export.js

@@ -38,7 +38,11 @@ $("#import-upload").change(async function() {
         .done(async note => {
             await treeService.reload();
 
-            await treeService.activateNote(note.noteId);
+            if (note) {
+                const node = await treeService.activateNote(note.noteId);
+
+                node.setExpanded(true);
+            }
         });
 });
 

src/public/javascripts/services/tree_context_menu.js

@@ -99,7 +99,7 @@ const contextMenuOptions = {
             {title: "OPML", cmd: "exportSubtreeToOpml"},
             {title: "Markdown", cmd: "exportSubtreeToMarkdown"}
         ]},
-        {title: "Import into note (tar, opml, md)", cmd: "importIntoNote", uiIcon: "ui-icon-arrowthick-1-sw"},
+        {title: "Import into note (tar, opml, md, enex)", cmd: "importIntoNote", uiIcon: "ui-icon-arrowthick-1-sw"},
         {title: "----"},
         {title: "Collapse subtree <kbd>Alt+-</kbd>", cmd: "collapseSubtree", uiIcon: "ui-icon-minus"},
         {title: "Force note sync", cmd: "forceNoteSync", uiIcon: "ui-icon-refresh"},

src/routes/api/file_upload.js

@@ -48,7 +48,7 @@ async function downloadFile(req, res) {
     }
 
     const originalFileName = await note.getLabel('originalFileName');
-    const fileName = originalFileName.value || note.title;
+    const fileName = originalFileName ? originalFileName.value : note.title;
 
     res.setHeader('Content-Disposition', 'file; filename="' + fileName + '"');
     res.setHeader('Content-Type', note.mime);

src/routes/api/import.js

@@ -2,6 +2,7 @@
 
 const repository = require('../../services/repository');
 const log = require('../../services/log');
+const enex = require('../../services/enex');
 const attributeService = require('../../services/attributes');
 const noteService = require('../../services/notes');
 const Branch = require('../../entities/branch');
@@ -28,13 +29,16 @@ async function importToBranch(req) {
     const extension = path.extname(file.originalname).toLowerCase();
 
     if (extension === '.tar') {
-        return await importTar(file, parentNoteId);
+        return await importTar(file, parentNote);
     }
     else if (extension === '.opml') {
-        return await importOpml(file, parentNoteId);
+        return await importOpml(file, parentNote);
     }
     else if (extension === '.md') {
-        return await importMarkdown(file, parentNoteId);
+        return await importMarkdown(file, parentNote);
+    }
+    else if (extension === '.enex') {
+        return await enex.importEnex(file, parentNote);
     }
     else {
         return [400, `Unrecognized extension ${extension}, must be .tar or .opml`];
@@ -59,7 +63,7 @@ async function importOutline(outline, parentNoteId) {
     return note;
 }
 
-async function importOpml(file, parentNoteId) {
+async function importOpml(file, parentNote) {
     const xml = await new Promise(function(resolve, reject)
     {
         parseString(file.buffer, function (err, result) {
@@ -80,7 +84,7 @@ async function importOpml(file, parentNoteId) {
     let returnNote = null;
 
     for (const outline of outlines) {
-        const note = await importOutline(outline, parentNoteId);
+        const note = await importOutline(outline, parentNote.noteId);
 
         // first created note will be activated after import
         returnNote = returnNote || note;
@@ -89,7 +93,7 @@ async function importOpml(file, parentNoteId) {
     return returnNote;
 }
 
-async function importTar(file, parentNoteId) {
+async function importTar(file, parentNote) {
     const files = await parseImportFile(file);
 
     const ctx = {
@@ -100,7 +104,7 @@ async function importTar(file, parentNoteId) {
         writer: new commonmark.HtmlRenderer()
     };
 
-    const note = await importNotes(ctx, files, parentNoteId);
+    const note = await importNotes(ctx, files, parentNote.noteId);
 
     // we save attributes after importing notes because we need to have all the relation
     // targets already existing
@@ -290,7 +294,7 @@ async function importNotes(ctx, files, parentNoteId) {
     return returnNote;
 }
 
-async function importMarkdown(file, parentNoteId) {
+async function importMarkdown(file, parentNote) {
     const markdownContent = file.buffer.toString("UTF-8");
 
     const reader = new commonmark.Parser();
@@ -301,7 +305,7 @@ async function importMarkdown(file, parentNoteId) {
 
     const title = file.originalname.substr(0, file.originalname.length - 3); // strip .md extension
 
-    const {note} = await noteService.createNote(parentNoteId, title, htmlContent, {
+    const {note} = await noteService.createNote(parentNote.noteId, title, htmlContent, {
         type: 'text',
         mime: 'text/html'
     });

src/services/enex.js

@@ -0,0 +1,247 @@
+const sax = require("sax");
+const stream = require('stream');
+const xml2js = require('xml2js');
+const log = require("./log");
+const utils = require("./utils");
+const noteService = require("./notes");
+
+// date format is e.g. 20181121T193703Z
+function parseDate(text) {
+    // insert - and : to make it ISO format
+    text = text.substr(0, 4) + "-" + text.substr(4, 2) + "-" + text.substr(6, 2)
+        + "T" + text.substr(9, 2) + ":" + text.substr(11, 2) + ":" + text.substr(13, 2) + "Z";
+
+    return text;
+}
+
+let note = {};
+let resource;
+
+async function importEnex(file, parentNote) {
+    const saxStream = sax.createStream(true);
+    const xmlBuilder = new xml2js.Builder({ headless: true });
+    const parser = new xml2js.Parser({ explicitArray: true });
+
+    // we're persisting notes as we parse the document, but these are run asynchronously and may not be finished
+    // when we finish parsing. We use this to be sure that all saving has been finished before returning successfully.
+    const saveNotePromises = [];
+
+    async function parseXml(text) {
+        return new Promise(function(resolve, reject)
+        {
+            parser.parseString(text, function (err, result) {
+                if (err) {
+                    reject(err);
+                }
+                else {
+                    resolve(result);
+                }
+            });
+        });
+    }
+
+    function extractContent(enNote) {
+        // [] thing is workaround for https://github.com/Leonidas-from-XIV/node-xml2js/issues/484
+        let content = xmlBuilder.buildObject([enNote]);
+        content = content.substr(3, content.length - 7).trim();
+
+        // workaround for https://github.com/ckeditor/ckeditor5-list/issues/116
+        content = content.replace(/<li>\s+<div>/g, "<li>");
+        content = content.replace(/<\/div>\s+<\/li>/g, "</li>");
+
+        // workaround for https://github.com/ckeditor/ckeditor5-list/issues/115
+        content = content.replace(/<ul>\s+<ul>/g, "<ul><li><ul>");
+        content = content.replace(/<\/li>\s+<ul>/g, "<ul>");
+        content = content.replace(/<\/ul>\s+<\/ul>/g, "</ul></li></ul>");
+        content = content.replace(/<\/ul>\s+<li>/g, "</ul></li><li>");
+
+        content = content.replace(/<ol>\s+<ol>/g, "<ol><li><ol>");
+        content = content.replace(/<\/li>\s+<ol>/g, "<ol>");
+        content = content.replace(/<\/ol>\s+<\/ol>/g, "</ol></li></ol>");
+        content = content.replace(/<\/ol>\s+<li>/g, "</ol></li><li>");
+
+        return content;
+    }
+
+
+    const path = [];
+
+    function getCurrentTag() {
+        if (path.length >= 1) {
+            return path[path.length - 1];
+        }
+    }
+
+    function getPreviousTag() {
+        if (path.length >= 2) {
+            return path[path.length - 2];
+        }
+    }
+
+    saxStream.on("error", e => {
+        // unhandled errors will throw, since this is a proper node
+        // event emitter.
+        log.error("error when parsing ENEX file: " + e);
+        // clear the error
+        this._parser.error = null;
+        this._parser.resume();
+    });
+
+    saxStream.on("text", text => {
+        const currentTag = getCurrentTag();
+        const previousTag = getPreviousTag();
+
+        if (previousTag === 'note-attributes') {
+            note.attributes.push({
+                type: 'label',
+                name: currentTag,
+                value: text
+            });
+        }
+        else if (previousTag === 'resource-attributes') {
+            if (currentTag === 'file-name') {
+                resource.attributes.push({
+                    type: 'label',
+                    name: 'originalFileName',
+                    value: text
+                });
+
+                resource.title = text;
+            }
+            else if (currentTag === 'source-url') {
+                resource.attributes.push({
+                    type: 'label',
+                    name: 'sourceUrl',
+                    value: text
+                });
+            }
+        }
+        else if (previousTag === 'resource') {
+            if (currentTag === 'data') {
+                text = text.replace(/\s/g, '');
+
+                resource.content = utils.fromBase64(text);
+
+                resource.attributes.push({
+                    type: 'label',
+                    name: 'fileSize',
+                    value: resource.content.length
+                });
+            }
+            else if (currentTag === 'mime') {
+                resource.mime = text;
+
+                if (text.startsWith("image/")) {
+                    resource.title = "image";
+
+                    // images don't have "file-name" tag so we'll create attribute here
+                    resource.attributes.push({
+                        type: 'label',
+                        name: 'originalFileName',
+                        value: resource.title + "." + text.substr(6) // extension from mime type
+                    });
+                }
+            }
+        }
+        else if (previousTag === 'note') {
+            if (currentTag === 'title') {
+                note.title = text;
+            } else if (currentTag === 'created') {
+                note.dateCreated = parseDate(text);
+            } else if (currentTag === 'updated') {
+                // updated is currently ignored since dateModified is updated automatically with each save
+            } else if (currentTag === 'tag') {
+                note.attributes.push({
+                    type: 'label',
+                    name: text,
+                    value: ''
+                })
+            }
+            // unknown tags are just ignored
+        }
+    });
+
+    saxStream.on("attribute", attr => {
+        // an attribute.  attr has "name" and "value"
+    });
+
+    saxStream.on("opentag", tag => {
+        path.push(tag.name);
+
+        if (tag.name === 'note') {
+            note = {
+                content: "",
+                // it's an array, not a key-value object because we don't know if attributes can be duplicated
+                attributes: [],
+                resources: []
+            };
+        }
+        else if (tag.name === 'resource') {
+            resource = {
+                title: "resource",
+                attributes: []
+            };
+
+            note.resources.push(resource);
+        }
+    });
+
+    async function saveNote() {
+        // make a copy because stream continues with the next async call and note gets overwritten
+        let {title, content, attributes, resources, dateCreated} = note;
+
+        const xmlObject = await parseXml(content);
+
+        // following is workaround for this issue: https://github.com/Leonidas-from-XIV/node-xml2js/issues/484
+        content = extractContent(xmlObject['en-note']);
+
+        const resp = await noteService.createNote(parentNote.noteId, title, content, {
+            attributes,
+            dateCreated,
+            type: 'text',
+            mime: 'text/html'
+        });
+
+        for (const resource of resources) {
+            await noteService.createNote(resp.note.noteId, resource.title, resource.content, {
+                attributes: resource.attributes,
+                type: 'file',
+                mime: resource.mime
+            });
+        }
+    }
+
+    saxStream.on("closetag", async tag => {
+        path.pop();
+
+        if (tag === 'note') {
+            saveNotePromises.push(saveNote());
+        }
+    });
+
+    saxStream.on("opencdata", () => {
+        //console.log("opencdata");
+    });
+
+    saxStream.on("cdata", text => {
+        note.content += text;
+    });
+
+    saxStream.on("closecdata", () => {
+        //console.log("closecdata");
+    });
+
+    return new Promise((resolve, reject) =>
+    {
+        // resolve only when we parse the whole document AND saving of all notes have been finished
+        // we resolve to parentNote because there's no single note to pick
+        saxStream.on("end", () => { Promise.all(saveNotePromises).then(() => resolve(parentNote)) });
+
+        const bufferStream = new stream.PassThrough();
+        bufferStream.end(file.buffer);
+
+        bufferStream.pipe(saxStream);
+    });
+}
+
+module.exports = { importEnex };

src/services/notes.js

@@ -114,7 +114,8 @@ async function createNote(parentNoteId, title, content = "", extraOptions = {})
         target: 'into',
         isProtected: !!extraOptions.isProtected,
         type: extraOptions.type,
-        mime: extraOptions.mime
+        mime: extraOptions.mime,
+        dateCreated: extraOptions.dateCreated
     };
 
     if (extraOptions.json && !noteData.type) {

src/test/enex/Export-stack.enex

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE en-export SYSTEM "http://xml.evernote.com/pub/evernote-export2.dtd">
+<en-export export-date="20181101T193909Z" application="Evernote/Windows" version="6.x">
+<note><title>Note</title><content><![CDATA[<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE en-note SYSTEM "http://xml.evernote.com/pub/enml2.dtd">
+
+<en-note><div>this is a note in a notebook in a stack</div></en-note>]]></content><created>20181101T193703Z</created><updated>20181101T193712Z</updated><note-attributes><author>Adam Zivner</author><source>desktop.win</source><source-application>evernote.win32</source-application></note-attributes></note></en-export>