Merge branch 'master' into stable

app.js

@@ -10,8 +10,6 @@ const FileStore = require('session-file-store')(session);
 const os = require('os');
 const sessionSecret = require('./services/session_secret');
 
-require('./services/ping_job');
-
 const app = express();
 
 // view engine setup

migrations/0057__add_foreign_keys.sql

@@ -0,0 +1,101 @@
+INSERT INTO notes (note_id, note_title, note_text, date_created, date_modified)
+    VALUES ('root', 'root', 'root', strftime('%Y-%m-%dT%H:%M:%S.000Z', 'now'), strftime('%Y-%m-%dT%H:%M:%S.000Z', 'now'));
+
+CREATE TABLE IF NOT EXISTS "notes_mig" (
+    `note_id`	TEXT NOT NULL,
+    `note_title`	TEXT,
+    `note_text`	TEXT,
+    `is_protected`	INT NOT NULL DEFAULT 0,
+    `is_deleted`	INT NOT NULL DEFAULT 0,
+    `date_created`	TEXT NOT NULL,
+    `date_modified`	TEXT NOT NULL,
+    PRIMARY KEY(`note_id`)
+);
+
+INSERT INTO notes_mig (note_id, note_title, note_text, is_protected, is_deleted, date_created, date_modified)
+    SELECT note_id, note_title, note_text, is_protected, is_deleted, date_created, date_modified FROM notes;
+
+DROP TABLE notes;
+ALTER TABLE notes_mig RENAME TO notes;
+
+CREATE INDEX `IDX_notes_is_deleted` ON `notes` (
+    `is_deleted`
+);
+
+CREATE TABLE IF NOT EXISTS "notes_tree_mig" (
+    `note_tree_id`	TEXT NOT NULL,
+    `note_id`	TEXT NOT NULL,
+    `parent_note_id`	TEXT NOT NULL,
+    `note_position`	INTEGER NOT NULL,
+    `prefix`	TEXT,
+    `is_expanded`	BOOLEAN,
+    `is_deleted`	INTEGER NOT NULL DEFAULT 0,
+    `date_modified`	TEXT NOT NULL,
+    FOREIGN KEY(note_id) REFERENCES notes(note_id),
+    FOREIGN KEY(parent_note_id) REFERENCES notes(note_id),
+    PRIMARY KEY(`note_tree_id`)
+);
+
+INSERT INTO notes_tree_mig (note_tree_id, note_id, parent_note_id, note_position, prefix, is_expanded, is_deleted, date_modified)
+    SELECT note_tree_id, note_id, note_pid, note_pos, prefix, is_expanded, is_deleted, date_modified FROM notes_tree;
+
+DROP TABLE notes_tree;
+ALTER TABLE notes_tree_mig RENAME TO notes_tree;
+
+CREATE INDEX `IDX_notes_tree_note_tree_id` ON `notes_tree` (
+    `note_tree_id`
+);
+CREATE INDEX `IDX_notes_tree_note_id_parent_note_id` ON `notes_tree` (
+    `note_id`,
+    `parent_note_id`
+);
+CREATE INDEX `IDX_notes_tree_note_id` ON `notes_tree` (
+    `note_id`
+);
+
+CREATE TABLE IF NOT EXISTS "notes_history_mig" (
+    `note_history_id`	TEXT NOT NULL PRIMARY KEY,
+    `note_id`	TEXT NOT NULL,
+    `note_title`	TEXT,
+    `note_text`	TEXT,
+    `is_protected`	INT NOT NULL DEFAULT 0,
+    `date_modified_from` TEXT NOT NULL,
+    `date_modified_to` TEXT NOT NULL,
+    FOREIGN KEY(note_id) REFERENCES notes(note_id)
+);
+
+INSERT INTO notes_history_mig (note_history_id, note_id, note_title, note_text, is_protected, date_modified_from, date_modified_to)
+    SELECT note_history_id, note_id, note_title, note_text, is_protected, date_modified_from, date_modified_to FROM notes_history;
+
+DROP TABLE notes_history;
+ALTER TABLE notes_history_mig RENAME TO notes_history;
+
+CREATE INDEX `IDX_notes_history_note_id` ON `notes_history` (
+    `note_id`
+);
+CREATE INDEX `IDX_notes_history_note_date_modified_from` ON `notes_history` (
+    `date_modified_from`
+);
+CREATE INDEX `IDX_notes_history_note_date_modified_to` ON `notes_history` (
+    `date_modified_to`
+);
+
+DROP TABLE recent_notes;
+
+CREATE TABLE `recent_notes` (
+    `note_tree_id` TEXT NOT NULL PRIMARY KEY,
+    `note_path` TEXT NOT NULL,
+    `date_accessed` TEXT NOT NULL,
+    is_deleted INT,
+    FOREIGN KEY(note_tree_id) REFERENCES notes_tree(note_tree_id)
+);
+
+DROP TABLE event_log;
+
+CREATE TABLE `event_log` (
+    `id`	INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+    `note_id`	TEXT,
+    `comment`	TEXT,
+    `date_added`	TEXT NOT NULL,
+    FOREIGN KEY(note_id) REFERENCES notes(note_id)
+);

package.json

@@ -1,7 +1,7 @@
 {
   "name": "trilium",
   "description": "Trilium",
-  "version": "0.0.6",
+  "version": "0.0.7",
   "scripts": {
     "start": "node ./bin/www",
     "test-electron": "xo",

public/javascripts/context_menu.js

@@ -88,7 +88,7 @@ const contextMenu = (function() {
             const node = $.ui.fancytree.getNode(ui.target);
 
             if (ui.cmd === "insertNoteHere") {
-                const parentNoteId = node.data.note_pid;
+                const parentNoteId = node.data.parent_note_id;
                 const isProtected = treeUtils.getParentProtectedStatus(node);
 
                 noteTree.createNote(node, parentNoteId, 'after', isProtected);

public/javascripts/dialogs/settings.js

@@ -154,6 +154,7 @@ settings.addModule((async function () {
 
 settings.addModule((async function () {
     const forceFullSyncButton = $("#force-full-sync-button");
+    const fillSyncRowsButton = $("#fill-sync-rows-button");
 
     forceFullSyncButton.click(async () => {
         await server.post('sync/force-full-sync');
@@ -161,6 +162,12 @@ settings.addModule((async function () {
         showMessage("Full sync triggered");
     });
 
+    fillSyncRowsButton.click(async () => {
+        await server.post('sync/fill-sync-rows');
+
+        showMessage("Sync rows filled successfully");
+    });
+
     return {};
 })());
 

public/javascripts/dialogs/sql_console.js

@@ -20,15 +20,22 @@ const sqlConsole = (function() {
     async function execute() {
         const sqlQuery = queryEl.val();
 
-        const results = await server.post("sql/execute", {
+        const result = await server.post("sql/execute", {
             query: sqlQuery
         });
 
+        if (!result.success) {
+            showError(result.error);
+            return;
+        }
+
+        const rows = result.rows;
+
         resultHeadEl.empty();
         resultBodyEl.empty();
 
-        if (results.length > 0) {
+        if (rows.length > 0) {
-            const result = results[0];
+            const result = rows[0];
             const rowEl = $("<tr>");
 
             for (const key in result) {
@@ -38,7 +45,7 @@ const sqlConsole = (function() {
             resultHeadEl.append(rowEl);
         }
 
-        for (const result of results) {
+        for (const result of rows) {
             const rowEl = $("<tr>");
 
             for (const key in result) {

public/javascripts/messaging.js

@@ -2,7 +2,6 @@
 
 const messaging = (function() {
     const changesToPushCountEl = $("#changes-to-push-count");
-    let ws = null;
 
     function logError(message) {
         console.log(now(), message); // needs to be separate from .trace()
@@ -21,12 +20,15 @@ const messaging = (function() {
 
         if (message.type === 'sync') {
             lastPingTs = new Date().getTime();
-            const syncData = message.data.filter(sync => sync.source_id !== glob.sourceId);
 
-            if (syncData.length > 0) {
+            if (message.data.length > 0) {
-                console.log(now(), "Sync data: ", message);
+                console.log(now(), "Sync data: ", message.data);
+
+                lastSyncId = message.data[message.data.length - 1].id;
             }
 
+            const syncData = message.data.filter(sync => sync.source_id !== glob.sourceId);
+
             if (syncData.some(sync => sync.entity_name === 'notes_tree')) {
                 console.log(now(), "Reloading tree because of background changes");
 
@@ -59,17 +61,20 @@ const messaging = (function() {
         const protocol = document.location.protocol === 'https:' ? 'wss' : 'ws';
 
         // use wss for secure messaging
-        ws = new WebSocket(protocol + "://" + location.host);
+        const ws = new WebSocket(protocol + "://" + location.host);
         ws.onopen = event => console.log(now(), "Connected to server with WebSocket");
         ws.onmessage = messageHandler;
         ws.onclose = function(){
             // Try to reconnect in 5 seconds
             setTimeout(() => connectWebSocket(), 5000);
         };
+
+        return ws;
     }
 
-    connectWebSocket();
+    const ws = connectWebSocket();
 
+    let lastSyncId = glob.maxSyncIdAtLoad;
     let lastPingTs = new Date().getTime();
     let connectionBrokenNotification = null;
 
@@ -92,7 +97,12 @@ const messaging = (function() {
 
             showMessage("Re-connected to server");
         }
-    }, 3000);
+
+        ws.send(JSON.stringify({
+            type: 'ping',
+            lastSyncId: lastSyncId
+        }));
+    }, 1000);
 
     return {
         logError

public/javascripts/note_tree.js

@@ -132,7 +132,7 @@ const noteTree = (function() {
 
             delete note.note_title; // this should not be used. Use noteIdToTitle instead
 
-            setParentChildRelation(note.note_tree_id, note.note_pid, note.note_id);
+            setParentChildRelation(note.note_tree_id, note.parent_note_id, note.note_id);
         }
 
         return prepareNoteTreeInner('root');
@@ -171,7 +171,7 @@ const noteTree = (function() {
 
             const node = {
                 note_id: noteTree.note_id,
-                note_pid: noteTree.note_pid,
+                parent_note_id: noteTree.parent_note_id,
                 note_tree_id: noteTree.note_tree_id,
                 is_protected: noteTree.is_protected,
                 prefix: noteTree.prefix,
@@ -207,7 +207,7 @@ const noteTree = (function() {
         //console.log(now(), "Run path: ", runPath);
 
         for (const childNoteId of runPath) {
-            const node = getNodesByNoteId(childNoteId).find(node => node.data.note_pid === parentNoteId);
+            const node = getNodesByNoteId(childNoteId).find(node => node.data.parent_note_id === parentNoteId);
 
             if (childNoteId === noteId) {
                 await node.setActive();
@@ -334,6 +334,10 @@ const noteTree = (function() {
         while (cur !== 'root') {
             path.push(cur);
 
+            if (!childToParents[cur]) {
+                throwError("Can't find parents for " + cur);
+            }
+
             cur = childToParents[cur][0];
         }
 
@@ -505,12 +509,16 @@ const noteTree = (function() {
         await getTree().reload(notes);
     }
 
+    function getNotePathFromAddress() {
+        return document.location.hash.substr(1); // strip initial #
+    }
+
     function loadTree() {
         return server.get('tree').then(resp => {
             startNotePath = resp.start_note_path;
 
             if (document.location.hash) {
-                startNotePath = document.location.hash.substr(1); // strip initial #
+                startNotePath = getNotePathFromAddress();
             }
 
             return prepareNoteTree(resp.notes);
@@ -610,7 +618,7 @@ const noteTree = (function() {
         const newNode = {
             title: newNoteName,
             note_id: result.note_id,
-            note_pid: parentNoteId,
+            parent_note_id: parentNoteId,
             refKey: result.note_id,
             note_tree_id: result.note_tree_id,
             is_protected: isProtected,
@@ -642,7 +650,7 @@ const noteTree = (function() {
         console.log("pressed O");
 
         const node = getCurrentNode();
-        const parentNoteId = node.data.note_pid;
+        const parentNoteId = node.data.parent_note_id;
         const isProtected = treeUtils.getParentProtectedStatus(node);
 
         createNote(node, parentNoteId, 'after', isProtected);
@@ -668,6 +676,26 @@ const noteTree = (function() {
 
     $(document).bind('keydown', 'ctrl+.', scrollToCurrentNote);
 
+    $(window).bind('hashchange', function() {
+        const notePath = getNotePathFromAddress();
+
+        activateNode(notePath);
+    });
+
+    if (isElectron()) {
+        $(document).bind('keydown', 'alt+left', e => {
+            window.history.back();
+
+            e.preventDefault();
+        });
+
+        $(document).bind('keydown', 'alt+right', e => {
+            window.history.forward();
+
+            e.preventDefault();
+        });
+    }
+
     return {
         reload,
         collapseTree,

public/javascripts/tree_changes.js

@@ -96,13 +96,13 @@ const treeChanges = (function() {
     }
 
     function changeNode(node, func) {
-        noteTree.removeParentChildRelation(node.data.note_pid, node.data.note_id);
+        noteTree.removeParentChildRelation(node.data.parent_note_id, node.data.note_id);
 
         func(node);
 
-        node.data.note_pid = node.getParent() === null ? 'root' : node.getParent().data.note_id;
+        node.data.parent_note_id = node.getParent() === null ? 'root' : node.getParent().data.note_id;
 
-        noteTree.setParentChildRelation(node.data.note_tree_id, node.data.note_pid, node.data.note_id);
+        noteTree.setParentChildRelation(node.data.note_tree_id, node.data.parent_note_id, node.data.note_id);
 
         noteTree.setCurrentNotePathToHash(node);
     }

routes/api/export.js

@@ -35,11 +35,11 @@ async function exportNote(noteTreeId, dir) {
     const noteTree = await sql.getSingleResult("SELECT * FROM notes_tree WHERE note_tree_id = ?", [noteTreeId]);
     const note = await sql.getSingleResult("SELECT * FROM notes WHERE note_id = ?", [noteTree.note_id]);
 
-    const pos = (noteTree.note_pos + '').padStart(4, '0');
+    const pos = (noteTree.note_position + '').padStart(4, '0');
 
     fs.writeFileSync(dir + '/' + pos + '-' + note.note_title + '.html', html.prettyPrint(note.note_text, {indent_size: 2}));
 
-    const children = await sql.getResults("SELECT * FROM notes_tree WHERE note_pid = ? AND is_deleted = 0", [note.note_id]);
+    const children = await sql.getResults("SELECT * FROM notes_tree WHERE parent_note_id = ? AND is_deleted = 0", [note.note_id]);
 
     if (children.length > 0) {
         const childrenDir = dir + '/' + pos + '-' + note.note_title;

routes/api/import.js

@@ -51,7 +51,7 @@ async function importNotes(dir, parentNoteId) {
             noteTitle = match[2];
         }
         else {
-            let maxPos = await sql.getSingleValue("SELECT MAX(note_pos) FROM notes_tree WHERE note_pid = ? AND is_deleted = 0", [parentNoteId]);
+            let maxPos = await sql.getSingleValue("SELECT MAX(note_position) FROM notes_tree WHERE parent_note_id = ? AND is_deleted = 0", [parentNoteId]);
             if (maxPos) {
                 notePos = maxPos + 1;
             }
@@ -72,8 +72,8 @@ async function importNotes(dir, parentNoteId) {
         await sql.insert('notes_tree', {
             note_tree_id: noteTreeId,
             note_id: noteId,
-            note_pid: parentNoteId,
+            parent_note_id: parentNoteId,
-            note_pos: notePos,
+            note_position: notePos,
             is_expanded: 0,
             is_deleted: 0,
             date_modified: now

routes/api/notes_move.js

@@ -12,13 +12,13 @@ router.put('/:noteTreeId/move-to/:parentNoteId', auth.checkApiAuth, async (req,
     const parentNoteId = req.params.parentNoteId;
     const sourceId = req.headers.source_id;
 
-    const maxNotePos = await sql.getSingleValue('SELECT MAX(note_pos) FROM notes_tree WHERE note_pid = ? AND is_deleted = 0', [parentNoteId]);
+    const maxNotePos = await sql.getSingleValue('SELECT MAX(note_position) FROM notes_tree WHERE parent_note_id = ? AND is_deleted = 0', [parentNoteId]);
     const newNotePos = maxNotePos === null ? 0 : maxNotePos + 1;
 
     const now = utils.nowDate();
 
     await sql.doInTransaction(async () => {
-        await sql.execute("UPDATE notes_tree SET note_pid = ?, note_pos = ?, date_modified = ? WHERE note_tree_id = ?",
+        await sql.execute("UPDATE notes_tree SET parent_note_id = ?, note_position = ?, date_modified = ? WHERE note_tree_id = ?",
             [parentNoteId, newNotePos, now, noteTreeId]);
 
         await sync_table.addNoteTreeSync(noteTreeId, sourceId);
@@ -38,15 +38,15 @@ router.put('/:noteTreeId/move-before/:beforeNoteTreeId', async (req, res, next)
         await sql.doInTransaction(async () => {
             // we don't change date_modified so other changes are prioritized in case of conflict
             // also we would have to sync all those modified note trees otherwise hash checks would fail
-            await sql.execute("UPDATE notes_tree SET note_pos = note_pos + 1 WHERE note_pid = ? AND note_pos >= ? AND is_deleted = 0",
+            await sql.execute("UPDATE notes_tree SET note_position = note_position + 1 WHERE parent_note_id = ? AND note_position >= ? AND is_deleted = 0",
-                [beforeNote.note_pid, beforeNote.note_pos]);
+                [beforeNote.parent_note_id, beforeNote.note_position]);
 
-            await sync_table.addNoteReorderingSync(beforeNote.note_pid, sourceId);
+            await sync_table.addNoteReorderingSync(beforeNote.parent_note_id, sourceId);
 
             const now = utils.nowDate();
 
-            await sql.execute("UPDATE notes_tree SET note_pid = ?, note_pos = ?, date_modified = ? WHERE note_tree_id = ?",
+            await sql.execute("UPDATE notes_tree SET parent_note_id = ?, note_position = ?, date_modified = ? WHERE note_tree_id = ?",
-                [beforeNote.note_pid, beforeNote.note_pos, now, noteTreeId]);
+                [beforeNote.parent_note_id, beforeNote.note_position, now, noteTreeId]);
 
             await sync_table.addNoteTreeSync(noteTreeId, sourceId);
         });
@@ -69,13 +69,13 @@ router.put('/:noteTreeId/move-after/:afterNoteTreeId', async (req, res, next) =>
         await sql.doInTransaction(async () => {
             // we don't change date_modified so other changes are prioritized in case of conflict
             // also we would have to sync all those modified note trees otherwise hash checks would fail
-            await sql.execute("UPDATE notes_tree SET note_pos = note_pos + 1 WHERE note_pid = ? AND note_pos > ? AND is_deleted = 0",
+            await sql.execute("UPDATE notes_tree SET note_position = note_position + 1 WHERE parent_note_id = ? AND note_position > ? AND is_deleted = 0",
-                [afterNote.note_pid, afterNote.note_pos]);
+                [afterNote.parent_note_id, afterNote.note_position]);
 
-            await sync_table.addNoteReorderingSync(afterNote.note_pid, sourceId);
+            await sync_table.addNoteReorderingSync(afterNote.parent_note_id, sourceId);
 
-            await sql.execute("UPDATE notes_tree SET note_pid = ?, note_pos = ?, date_modified = ? WHERE note_tree_id = ?",
+            await sql.execute("UPDATE notes_tree SET parent_note_id = ?, note_position = ?, date_modified = ? WHERE note_tree_id = ?",
-                [afterNote.note_pid, afterNote.note_pos + 1, utils.nowDate(), noteTreeId]);
+                [afterNote.parent_note_id, afterNote.note_position + 1, utils.nowDate(), noteTreeId]);
 
             await sync_table.addNoteTreeSync(noteTreeId, sourceId);
         });
@@ -92,7 +92,7 @@ router.put('/:childNoteId/clone-to/:parentNoteId', auth.checkApiAuth, async (req
     const childNoteId = req.params.childNoteId;
     const sourceId = req.headers.source_id;
 
-    const existing = await sql.getSingleValue('SELECT * FROM notes_tree WHERE note_id = ? AND note_pid = ?', [childNoteId, parentNoteId]);
+    const existing = await sql.getSingleValue('SELECT * FROM notes_tree WHERE note_id = ? AND parent_note_id = ?', [childNoteId, parentNoteId]);
 
     if (existing && !existing.is_deleted) {
         return res.send({
@@ -108,15 +108,15 @@ router.put('/:childNoteId/clone-to/:parentNoteId', auth.checkApiAuth, async (req
         });
     }
 
-    const maxNotePos = await sql.getSingleValue('SELECT MAX(note_pos) FROM notes_tree WHERE note_pid = ? AND is_deleted = 0', [parentNoteId]);
+    const maxNotePos = await sql.getSingleValue('SELECT MAX(note_position) FROM notes_tree WHERE parent_note_id = ? AND is_deleted = 0', [parentNoteId]);
     const newNotePos = maxNotePos === null ? 0 : maxNotePos + 1;
 
     await sql.doInTransaction(async () => {
         const noteTree = {
             note_tree_id: utils.newNoteTreeId(),
             note_id: childNoteId,
-            note_pid: parentNoteId,
+            parent_note_id: parentNoteId,
-            note_pos: newNotePos,
+            note_position: newNotePos,
             is_expanded: 0,
             date_modified: utils.nowDate(),
             is_deleted: 0
@@ -143,14 +143,14 @@ router.put('/:noteId/clone-after/:afterNoteTreeId', async (req, res, next) => {
         return res.status(500).send("After note " + afterNoteTreeId + " doesn't exist.");
     }
 
-    if (!await checkCycle(afterNote.note_pid, noteId)) {
+    if (!await checkCycle(afterNote.parent_note_id, noteId)) {
         return res.send({
             success: false,
             message: 'Cloning note here would create cycle.'
         });
     }
 
-    const existing = await sql.getSingleValue('SELECT * FROM notes_tree WHERE note_id = ? AND note_pid = ?', [noteId, afterNote.note_pid]);
+    const existing = await sql.getSingleValue('SELECT * FROM notes_tree WHERE note_id = ? AND parent_note_id = ?', [noteId, afterNote.parent_note_id]);
 
     if (existing && !existing.is_deleted) {
         return res.send({
@@ -162,16 +162,16 @@ router.put('/:noteId/clone-after/:afterNoteTreeId', async (req, res, next) => {
     await sql.doInTransaction(async () => {
         // we don't change date_modified so other changes are prioritized in case of conflict
         // also we would have to sync all those modified note trees otherwise hash checks would fail
-        await sql.execute("UPDATE notes_tree SET note_pos = note_pos + 1 WHERE note_pid = ? AND note_pos > ? AND is_deleted = 0",
+        await sql.execute("UPDATE notes_tree SET note_position = note_position + 1 WHERE parent_note_id = ? AND note_position > ? AND is_deleted = 0",
-            [afterNote.note_pid, afterNote.note_pos]);
+            [afterNote.parent_note_id, afterNote.note_position]);
 
-        await sync_table.addNoteReorderingSync(afterNote.note_pid, sourceId);
+        await sync_table.addNoteReorderingSync(afterNote.parent_note_id, sourceId);
 
         const noteTree = {
             note_tree_id: utils.newNoteTreeId(),
             note_id: noteId,
-            note_pid: afterNote.note_pid,
+            parent_note_id: afterNote.parent_note_id,
-            note_pos: afterNote.note_pos + 1,
+            note_position: afterNote.note_position + 1,
             is_expanded: 0,
             date_modified: utils.nowDate(),
             is_deleted: 0
@@ -196,7 +196,7 @@ async function checkCycle(parentNoteId, childNoteId) {
         return false;
     }
 
-    const parentNoteIds = await sql.getFlattenedResults("SELECT DISTINCT note_pid FROM notes_tree WHERE note_id = ?", [parentNoteId]);
+    const parentNoteIds = await sql.getFlattenedResults("SELECT DISTINCT parent_note_id FROM notes_tree WHERE note_id = ?", [parentNoteId]);
 
     for (const pid of parentNoteIds) {
         if (!await checkCycle(pid, childNoteId)) {

routes/api/sql.js

@@ -8,7 +8,18 @@ const sql = require('../../services/sql');
 router.post('/execute', auth.checkApiAuth, async (req, res, next) => {
     const query = req.body.query;
 
-    res.send(await sql.getResults(query));
+    try {
+        res.send({
+            success: true,
+            rows: await sql.getResults(query)
+        });
+    }
+    catch (e) {
+        res.send({
+            success: false,
+            error: e.message
+        });
+    }
 });
 
 module.exports = router;

routes/api/sync.js

@@ -8,6 +8,8 @@ const syncUpdate = require('../../services/sync_update');
 const sql = require('../../services/sql');
 const options = require('../../services/options');
 const content_hash = require('../../services/content_hash');
+const utils = require('../../services/utils');
+const log = require('../../services/log');
 
 router.get('/check', auth.checkApiAuth, async (req, res, next) => {
     res.send({
@@ -20,6 +22,44 @@ router.post('/now', auth.checkApiAuth, async (req, res, next) => {
     res.send(await sync.sync());
 });
 
+async function fillSyncRows(entityName, entityKey) {
+    // cleanup sync rows for missing entities
+    await sql.execute(`
+      DELETE 
+      FROM sync 
+      WHERE sync.entity_name = '${entityName}' 
+        AND sync.entity_id NOT IN (SELECT ${entityKey} FROM ${entityName})`);
+
+    const entityIds = await sql.getFlattenedResults(`SELECT ${entityKey} FROM ${entityName}`);
+
+    for (const entityId of entityIds) {
+        const existingRows = await sql.getSingleValue("SELECT COUNT(id) FROM sync WHERE entity_name = ? AND entity_id = ?", [entityName, entityId]);
+
+        // we don't want to replace existing entities (which would effectively cause full resync)
+        if (existingRows === 0) {
+            log.info(`Creating missing sync record for ${entityName} ${entityId}`);
+
+            await sql.insert("sync", {
+                entity_name: entityName,
+                entity_id: entityId,
+                source_id: "SYNC_FILL",
+                sync_date: utils.nowDate()
+            });
+        }
+    }
+}
+
+router.post('/fill-sync-rows', auth.checkApiAuth, async (req, res, next) => {
+    await sql.doInTransaction(async () => {
+        await fillSyncRows("notes", "note_id");
+        await fillSyncRows("notes_tree", "note_tree_id");
+        await fillSyncRows("notes_history", "note_history_id");
+        await fillSyncRows("recent_notes", "note_tree_id");
+    });
+
+    res.send({});
+});
+
 router.post('/force-full-sync', auth.checkApiAuth, async (req, res, next) => {
     await sql.doInTransaction(async () => {
         await options.setOption('last_synced_pull', 0);
@@ -73,8 +113,8 @@ router.get('/notes_reordering/:noteTreeParentId', auth.checkApiAuth, async (req,
     const noteTreeParentId = req.params.noteTreeParentId;
 
     res.send({
-        note_pid: noteTreeParentId,
+        parent_note_id: noteTreeParentId,
-        ordering: await sql.getMap("SELECT note_tree_id, note_pos FROM notes_tree WHERE note_pid = ?", [noteTreeParentId])
+        ordering: await sql.getMap("SELECT note_tree_id, note_position FROM notes_tree WHERE parent_note_id = ?", [noteTreeParentId])
     });
 });
 

routes/api/tree.js

@@ -19,7 +19,7 @@ router.get('/', auth.checkApiAuth, async (req, res, next) => {
         + "FROM notes_tree "
         + "JOIN notes ON notes.note_id = notes_tree.note_id "
         + "WHERE notes.is_deleted = 0 AND notes_tree.is_deleted = 0 "
-        + "ORDER BY note_pos");
+        + "ORDER BY note_position");
 
     const dataKey = protected_session.getDataKey(req);
 

routes/index.js

@@ -4,10 +4,12 @@ const express = require('express');
 const router = express.Router();
 const auth = require('../services/auth');
 const source_id = require('../services/source_id');
+const sql = require('../services/sql');
 
 router.get('', auth.checkAuth, async (req, res, next) => {
     res.render('index', {
-        sourceId: await source_id.generateSourceId()
+        sourceId: await source_id.generateSourceId(),
+        maxSyncIdAtLoad: await sql.getSingleValue("SELECT MAX(id) FROM sync")
     });
 });
 

schema.sql

@@ -70,8 +70,8 @@ CREATE TABLE `recent_notes` (
 CREATE TABLE IF NOT EXISTS "notes_tree" (
   `note_tree_id`	TEXT NOT NULL,
   `note_id`	TEXT NOT NULL,
-  `note_pid`	TEXT NOT NULL,
+  `parent_note_id`	TEXT NOT NULL,
-  `note_pos`	INTEGER NOT NULL,
+  `note_position`	INTEGER NOT NULL,
   `prefix`	TEXT,
   `is_expanded`	BOOLEAN,
   `is_deleted`	INTEGER NOT NULL DEFAULT 0,
@@ -81,7 +81,7 @@ CREATE TABLE IF NOT EXISTS "notes_tree" (
 CREATE INDEX `IDX_notes_tree_note_id` ON `notes_tree` (
   `note_id`
 );
-CREATE INDEX `IDX_notes_tree_note_id_note_pid` ON `notes_tree` (
+CREATE INDEX `IDX_notes_tree_note_id_parent_note_id` ON `notes_tree` (
   `note_id`,
-  `note_pid`
+  `parent_note_id`
 );

services/app_info.js

@@ -3,7 +3,7 @@
 const build = require('./build');
 const packageJson = require('../package');
 
-const APP_DB_VERSION = 56;
+const APP_DB_VERSION = 57;
 
 module.exports = {
     app_version: packageJson.version,

services/build.js

@@ -1 +1 @@
-module.exports = { build_date:"2017-12-18T00:01:16-05:00", build_revision: "f96e38fd13152eee4700ead265c5b255b8e6853e" };
+module.exports = { build_date:"2017-12-18T23:45:10-05:00", build_revision: "b0e2d99a7b1073e9ee593b386afa19a62a2651eb" };

services/consistency_checks.js

@@ -26,7 +26,7 @@ async function runSyncRowChecks(table, key, errorList) {
 async function runChecks() {
     const errorList = [];
 
-    await runCheck("SELECT note_id FROM notes LEFT JOIN notes_tree USING(note_id) WHERE notes_tree.note_tree_id IS NULL",
+    await runCheck("SELECT note_id FROM notes LEFT JOIN notes_tree USING(note_id) WHERE note_id != 'root' AND notes_tree.note_tree_id IS NULL",
         "Missing notes_tree records for following note IDs", errorList);
 
     await runCheck("SELECT note_tree_id || ' > ' || notes_tree.note_id FROM notes_tree LEFT JOIN notes USING(note_id) WHERE notes.note_id IS NULL",
@@ -35,7 +35,7 @@ async function runChecks() {
     await runCheck("SELECT note_tree_id FROM notes_tree JOIN notes USING(note_id) WHERE notes.is_deleted = 1 AND notes_tree.is_deleted = 0",
         "Note tree is not deleted even though main note is deleted for following note tree IDs", errorList);
 
-    await runCheck("SELECT child.note_pid || ' > ' || child.note_id FROM notes_tree AS child LEFT JOIN notes_tree AS parent ON parent.note_id = child.note_pid WHERE parent.note_id IS NULL AND child.note_pid != 'root'",
+    await runCheck("SELECT child.parent_note_id || ' > ' || child.note_id FROM notes_tree AS child LEFT JOIN notes_tree AS parent ON parent.note_id = child.parent_note_id WHERE parent.note_id IS NULL AND child.parent_note_id != 'root'",
         "Not existing parent in the following parent > child relations", errorList);
 
     await runCheck("SELECT note_history_id || ' > ' || notes_history.note_id FROM notes_history LEFT JOIN notes USING(note_id) WHERE notes.note_id IS NULL",
@@ -47,7 +47,7 @@ async function runChecks() {
     await runSyncRowChecks("recent_notes", "note_tree_id", errorList);
 
     if (errorList.length > 0) {
-        messaging.sendMessage({type: 'consistency-checks-failed'});
+        messaging.sendMessageToAllClients({type: 'consistency-checks-failed'});
     }
     else {
         log.info("All consistency checks passed.");

services/content_hash.js

@@ -29,8 +29,8 @@ async function getHashes() {
         notes_tree: getHash(await sql.getResults(`SELECT
                                                        note_tree_id,
                                                        note_id,
-                                                       note_pid,
+                                                       parent_note_id,
-                                                       note_pos,
+                                                       note_position,
                                                        date_modified,
                                                        is_deleted,
                                                        prefix

services/messaging.js

@@ -1,6 +1,9 @@
 const WebSocket = require('ws');
 const utils = require('./utils');
 const log = require('./log');
+const sql = require('./sql');
+const options = require('./options');
+const sync_setup = require('./sync_setup');
 
 let webSocketServer;
 
@@ -29,6 +32,9 @@ function init(httpServer, sessionParser) {
             if (message.type === 'log-error') {
                 log.error('JS Error: ' + message.error);
             }
+            else if (message.type === 'ping') {
+                sendPing(ws, message.lastSyncId);
+            }
             else {
                 log.error('Unrecognized message: ');
                 log.error(message);
@@ -37,7 +43,15 @@ function init(httpServer, sessionParser) {
     });
 }
 
-async function sendMessage(message) {
+async function sendMessage(client, message) {
+    const jsonStr = JSON.stringify(message);
+
+    if (client.readyState === WebSocket.OPEN) {
+        client.send(jsonStr);
+    }
+}
+
+async function sendMessageToAllClients(message) {
     const jsonStr = JSON.stringify(message);
 
     webSocketServer.clients.forEach(function each(client) {
@@ -47,7 +61,21 @@ async function sendMessage(message) {
     });
 }
 
+async function sendPing(client, lastSentSyncId) {
+    const syncData = await sql.getResults("SELECT * FROM sync WHERE id > ?", [lastSentSyncId]);
+
+    const lastSyncedPush = await options.getOption('last_synced_push');
+
+    const changesToPushCount = await sql.getSingleValue("SELECT COUNT(*) FROM sync WHERE id > ?", [lastSyncedPush]);
+
+    await sendMessage(client, {
+        type: 'sync',
+        data: syncData,
+        changesToPushCount: sync_setup.isSyncSetup ? changesToPushCount : 0
+    });
+}
+
 module.exports = {
     init,
-    sendMessage
+    sendMessageToAllClients
 };

services/notes.js

@@ -13,18 +13,18 @@ async function createNewNote(parentNoteId, note, sourceId) {
 
     await sql.doInTransaction(async () => {
         if (note.target === 'into') {
-            const maxNotePos = await sql.getSingleValue('SELECT MAX(note_pos) FROM notes_tree WHERE note_pid = ? AND is_deleted = 0', [parentNoteId]);
+            const maxNotePos = await sql.getSingleValue('SELECT MAX(note_position) FROM notes_tree WHERE parent_note_id = ? AND is_deleted = 0', [parentNoteId]);
 
             newNotePos = maxNotePos === null ? 0 : maxNotePos + 1;
         }
         else if (note.target === 'after') {
-            const afterNote = await sql.getSingleResult('SELECT note_pos FROM notes_tree WHERE note_tree_id = ?', [note.target_note_tree_id]);
+            const afterNote = await sql.getSingleResult('SELECT note_position FROM notes_tree WHERE note_tree_id = ?', [note.target_note_tree_id]);
 
-            newNotePos = afterNote.note_pos + 1;
+            newNotePos = afterNote.note_position + 1;
 
             // not updating date_modified to avoig having to sync whole rows
-            await sql.execute('UPDATE notes_tree SET note_pos = note_pos + 1 WHERE note_pid = ? AND note_pos > ? AND is_deleted = 0',
+            await sql.execute('UPDATE notes_tree SET note_position = note_position + 1 WHERE parent_note_id = ? AND note_position > ? AND is_deleted = 0',
-                [parentNoteId, afterNote.note_pos]);
+                [parentNoteId, afterNote.note_position]);
 
             await sync_table.addNoteReorderingSync(parentNoteId, sourceId);
         }
@@ -48,8 +48,8 @@ async function createNewNote(parentNoteId, note, sourceId) {
         await sql.insert("notes_tree", {
             note_tree_id: noteTreeId,
             note_id: noteId,
-            note_pid: parentNoteId,
+            parent_note_id: parentNoteId,
-            note_pos: newNotePos,
+            note_position: newNotePos,
             is_expanded: 0,
             date_modified: now,
             is_deleted: 0
@@ -74,7 +74,7 @@ async function protectNoteRecursively(noteId, dataKey, protect, sourceId) {
 
     await protectNote(note, dataKey, protect, sourceId);
 
-    const children = await sql.getFlattenedResults("SELECT note_id FROM notes_tree WHERE note_pid = ?", [noteId]);
+    const children = await sql.getFlattenedResults("SELECT note_id FROM notes_tree WHERE parent_note_id = ?", [noteId]);
 
     for (const childNoteId of children) {
         await protectNoteRecursively(childNoteId, dataKey, protect, sourceId);
@@ -205,7 +205,7 @@ async function deleteNote(noteTreeId, sourceId) {
         await sql.execute("UPDATE notes SET is_deleted = 1, date_modified = ? WHERE note_id = ?", [now, noteId]);
         await sync_table.addNoteSync(noteId, sourceId);
 
-        const children = await sql.getResults("SELECT note_tree_id FROM notes_tree WHERE note_pid = ? AND is_deleted = 0", [noteId]);
+        const children = await sql.getResults("SELECT note_tree_id FROM notes_tree WHERE parent_note_id = ? AND is_deleted = 0", [noteId]);
 
         for (const child of children) {
             await deleteNote(child.note_tree_id, sourceId);

services/ping_job.js

@@ -1,30 +0,0 @@
-const sql = require('./sql');
-const messaging = require('./messaging');
-const options = require('./options');
-const sync_setup = require('./sync_setup');
-
-let lastSentSyncId;
-
-async function sendPing() {
-    const syncData = await sql.getResults("SELECT * FROM sync WHERE id > ?", [lastSentSyncId]);
-
-    const lastSyncedPush = await options.getOption('last_synced_push');
-
-    const changesToPushCount = await sql.getSingleValue("SELECT COUNT(*) FROM sync WHERE id > ?", [lastSyncedPush]);
-
-    messaging.sendMessage({
-        type: 'sync',
-        data: syncData,
-        changesToPushCount: sync_setup.isSyncSetup ? changesToPushCount : 0
-    });
-
-    if (syncData.length > 0) {
-        lastSentSyncId = syncData[syncData.length - 1].id;
-    }
-}
-
-sql.dbReady.then(async () => {
-    lastSentSyncId = await sql.getSingleValue("SELECT MAX(id) FROM sync");
-
-    setInterval(sendPing, 1000);
-});

services/sql.js

@@ -16,6 +16,8 @@ const dbConnected = createConnection();
 let dbReadyResolve = null;
 const dbReady = new Promise((resolve, reject) => {
     dbConnected.then(async db => {
+        await execute("PRAGMA foreign_keys = ON");
+
         dbReadyResolve = () => {
             log.info("DB ready.");
 
@@ -37,8 +39,8 @@ const dbReady = new Promise((resolve, reject) => {
                 await insert('notes_tree', {
                     note_tree_id: utils.newNoteTreeId(),
                     note_id: noteId,
-                    note_pid: 'root',
+                    parent_note_id: 'root',
-                    note_pos: 1,
+                    note_position: 1,
                     is_deleted: 0,
                     date_modified: now
                 });
@@ -190,6 +192,8 @@ async function wrap(func) {
     catch (e) {
         log.error("Error executing query. Inner exception: " + e.stack + thisError.stack);
 
+        thisError.message = e.stack;
+
         throw thisError;
     }
 }

services/sync.js

@@ -209,8 +209,8 @@ async function pushEntity(sync, syncContext) {
     }
     else if (sync.entity_name === 'notes_reordering') {
         entity = {
-            note_pid: sync.entity_id,
+            parent_note_id: sync.entity_id,
-            ordering: await sql.getMap('SELECT note_tree_id, note_pos FROM notes_tree WHERE note_pid = ?', [sync.entity_id])
+            ordering: await sql.getMap('SELECT note_tree_id, note_position FROM notes_tree WHERE parent_note_id = ?', [sync.entity_id])
         };
     }
     else if (sync.entity_name === 'options') {
@@ -267,7 +267,7 @@ async function checkContentHash(syncContext) {
 
             if (key !== 'recent_notes') {
                 // let's not get alarmed about recent notes which get updated often and can cause failures in race conditions
-                await messaging.sendMessage({type: 'sync-hash-check-failed'});
+                await messaging.sendMessageToAllClients({type: 'sync-hash-check-failed'});
             }
         }
     }

services/sync_update.js

@@ -53,10 +53,10 @@ async function updateNoteHistory(entity, sourceId) {
 async function updateNoteReordering(entity, sourceId) {
     await sql.doInTransaction(async () => {
         Object.keys(entity.ordering).forEach(async key => {
-            await sql.execute("UPDATE notes_tree SET note_pos = ? WHERE note_tree_id = ?", [entity.ordering[key], key]);
+            await sql.execute("UPDATE notes_tree SET note_position = ? WHERE note_tree_id = ?", [entity.ordering[key], key]);
         });
 
-        await sync_table.addNoteReorderingSync(entity.note_pid, sourceId);
+        await sync_table.addNoteReorderingSync(entity.parent_note_id, sourceId);
     });
 }
 

views/index.ejs

@@ -219,6 +219,11 @@
         </div>
         <div id="sync">
           <button id="force-full-sync-button" class="btn btn-sm">Force full sync</button>
+
+          <br/>
+          <br/>
+
+          <button id="fill-sync-rows-button" class="btn btn-sm">Fill sync rows</button>
         </div>
         <div id="debugging">
           <button id="anonymize-button" class="btn btn-sm">Save anonymized database</button>
@@ -296,7 +301,8 @@
       const baseApiUrl = 'api/';
       const glob = {
           activeDialog: null,
-          sourceId: '<%= sourceId %>'
+          sourceId: '<%= sourceId %>',
+          maxSyncIdAtLoad: <%= maxSyncIdAtLoad %>
       };
     </script>