#98, sync setup now doesn't copy the whole DB file, but sets up minimal database and starts off sync

db/migrations/0102__fix_sync_entityIds.sql

@@ -0,0 +1,2 @@
+DELETE FROM sync WHERE entityName = 'note_tree';
+DELETE FROM sync WHERE entityName = 'attributes';

package.json

@@ -61,7 +61,6 @@
     "simple-node-logger": "^0.93.37",
     "sqlite": "^2.9.2",
     "tar-stream": "^1.6.1",
-    "tmp-promise": "^1.0.5",
     "unescape": "^1.0.1",
     "ws": "^5.2.1",
     "xml2js": "^0.4.19"

src/public/javascripts/setup.js

@@ -34,7 +34,7 @@ function SetupModel() {
         this.setupSyncFromDesktop(false);
     };
 
-    this.finish = () => {
+    this.finish = async () => {
         if (this.setupNewDocument()) {
             const username = this.username();
             const password1 = this.password1();
@@ -84,20 +84,33 @@ function SetupModel() {
             }
 
             // not using server.js because it loads too many dependencies
-            $.post('/api/setup/sync-from-server', {
+            const resp = await $.post('/api/setup/sync-from-server', {
                 serverAddress: serverAddress,
                 username: username,
                 password: password
-            }).then(() => {
-                window.location.replace("/");
-            }).catch((err) => {
-                alert("Error, see dev console for details.");
-                console.error(err);
             });
+
+            if (resp.result === 'success') {
+                this.step('sync-in-progress');
+
+                checkOutstandingSyncs();
+
+                setInterval(checkOutstandingSyncs, 1000);
+            }
+            else {
+                showAlert('Sync setup failed: ', resp.error);
+            }
         }
     };
 }
 
+async function checkOutstandingSyncs() {
+    const stats = await $.get('/api/sync/stats');
+    const totalOutstandingSyncs = stats.outstandingPushes + stats.outstandingPulls;
+
+    $("#outstanding-syncs").html(totalOutstandingSyncs);
+}
+
 function showAlert(message) {
     $("#alert").html(message);
     $("#alert").show();

src/routes/api/setup.js

@@ -2,14 +2,10 @@
 
 const sqlInit = require('../../services/sql_init');
 const sql = require('../../services/sql');
-const cls = require('../../services/cls');
+const rp = require('request-promise');
-const tmp = require('tmp-promise');
+const Option = require('../../entities/option');
-const http = require('http');
+const syncService = require('../../services/sync');
-const fs = require('fs');
 const log = require('../../services/log');
-const DOCUMENT_PATH = require('../../services/data_dir').DOCUMENT_PATH;
-const sourceIdService = require('../../services/source_id');
-const url = require('url');
 
 async function setupNewDocument(req) {
     const { username, password } = req.body;
@@ -20,52 +16,44 @@ async function setupNewDocument(req) {
 async function setupSyncFromServer(req) {
     const { serverAddress, username, password } = req.body;
 
-    const tempFile = await tmp.file();
+    try {
+        log.info("Getting document options from sync server.");
 
-    await new Promise((resolve, reject) => {
+        // response is expected to contain documentId and documentSecret options
-        const file = fs.createWriteStream(tempFile.path);
+        const options = await rp.get({
-        const parsedAddress = url.parse(serverAddress);
+            uri: serverAddress + '/api/sync/document',
+            auth: {
+                'user': username,
+                'pass': password
+            },
+            json: true
+        });
 
-        const options = {
+        log.info("Creating database for sync");
-            method: 'GET',
+
-            protocol: parsedAddress.protocol,
+        await sql.transactional(async () => {
-            host: parsedAddress.hostname,
+            await sqlInit.createDatabaseForSync(serverAddress);
-            port: parsedAddress.port,
+
-            path: '/api/sync/document',
+            for (const opt of options) {
-            auth: username + ':' + password
+                await new Option(opt).save();
+            }
+        });
+
+        log.info("Triggering sync.");
+
+        // it's ok to not wait for it here
+        syncService.sync();
+
+        return { result: 'success' };
+    }
+    catch (e) {
+        log.error("Sync failed: " + e.message);
+
+        return {
+            result: 'failure',
+            error: e.message
         };
-
+    }
-        log.info("Getting document from: " + serverAddress);
-
-        http.request(options, function(response) {
-            response.pipe(file);
-
-            file.on('finish', function() {
-                log.info("Document download finished, closing & renaming.");
-
-                file.close(() => { // close() is async, call after close completes.
-                    fs.rename(tempFile.path, DOCUMENT_PATH, async () => {
-                        cls.reset();
-
-                        await sqlInit.initDbConnection();
-
-                        // we need to generate new source ID for this instance, otherwise it will
-                        // match the original server one
-                        await sql.transactional(async () => {
-                            await sourceIdService.generateSourceId();
-                        });
-
-                        resolve();
-                    });
-                });
-            });
-        }).on('error', function(err) { // Handle errors
-            fs.unlink(tempFile.path); // Delete the file async. (But we don't check the result)
-
-            reject(err.message);
-            log.error(err.message);
-        }).end();
-    });
 }
 
 module.exports = {

src/routes/api/sync.js

@@ -7,7 +7,7 @@ const sql = require('../../services/sql');
 const optionService = require('../../services/options');
 const contentHashService = require('../../services/content_hash');
 const log = require('../../services/log');
-const DOCUMENT_PATH = require('../../services/data_dir').DOCUMENT_PATH;
+const repository = require('../../services/repository');
 
 async function testSync() {
     try {
@@ -23,6 +23,10 @@ async function testSync() {
     }
 }
 
+async function getStats() {
+    return syncService.stats;
+}
+
 async function checkSync() {
     return {
         hashes: await contentHashService.getHashes(),
@@ -75,7 +79,10 @@ async function getChanged(req) {
 
     const syncs = await sql.getRows("SELECT * FROM sync WHERE id > ? LIMIT 1000", [lastSyncId]);
 
-    return await syncService.getSyncRecords(syncs);
+    return {
+        syncs: await syncService.getSyncRecords(syncs),
+        maxSyncId: await sql.getValue('SELECT MAX(id) FROM sync')
+    };
 }
 
 async function update(req) {
@@ -87,10 +94,13 @@ async function update(req) {
     }
 }
 
-async function getDocument(req, resp) {
+async function getDocument() {
-    log.info("Serving document.");
+    log.info("Serving document options.");
 
-    resp.sendFile(DOCUMENT_PATH);
+    return [
+        await repository.getOption('documentId'),
+        await repository.getOption('documentSecret')
+    ];
 }
 
 module.exports = {
@@ -102,5 +112,6 @@ module.exports = {
     forceNoteSync,
     getChanged,
     update,
-    getDocument
+    getDocument,
+    getStats
 };

src/routes/routes.js

@@ -156,7 +156,8 @@ function register(app) {
     apiRoute(POST, '/api/sync/force-note-sync/:noteId', syncApiRoute.forceNoteSync);
     apiRoute(GET, '/api/sync/changed', syncApiRoute.getChanged);
     apiRoute(PUT, '/api/sync/update', syncApiRoute.update);
-    route(GET, '/api/sync/document', [auth.checkBasicAuth], syncApiRoute.getDocument);
+    route(GET, '/api/sync/document', [auth.checkBasicAuth], syncApiRoute.getDocument, apiResultHandler);
+    route(GET, '/api/sync/stats', [], syncApiRoute.getStats, apiResultHandler);
 
     apiRoute(GET, '/api/event-log', eventLogRoute.getEventLog);
 

src/services/app_info.js

@@ -3,7 +3,7 @@
 const build = require('./build');
 const packageJson = require('../../package');
 
-const APP_DB_VERSION = 101;
+const APP_DB_VERSION = 102;
 const SYNC_VERSION = 1;
 
 module.exports = {

src/services/note_cache.js

@@ -5,6 +5,7 @@ const repository = require('./repository');
 const protectedSessionService = require('./protected_session');
 const utils = require('./utils');
 
+let loaded = false;
 let noteTitles;
 let protectedNoteTitles;
 let noteIds;
@@ -34,6 +35,8 @@ async function load() {
     for (const noteId of hiddenLabels) {
         archived[noteId] = true;
     }
+
+    loaded = true;
 }
 
 function findNotes(query) {
@@ -226,6 +229,10 @@ function getNotePath(noteId) {
 }
 
 eventService.subscribe(eventService.ENTITY_CHANGED, async ({entityName, entityId}) => {
+    if (!loaded) {
+        return;
+    }
+
     if (entityName === 'notes') {
         const note = await repository.getNote(entityId);
 
@@ -277,6 +284,10 @@ eventService.subscribe(eventService.ENTITY_CHANGED, async ({entityName, entityId
 });
 
 eventService.subscribe(eventService.ENTER_PROTECTED_SESSION, async () => {
+    if (!loaded) {
+        return;
+    }
+
     protectedNoteTitles = await sql.getMap(`SELECT noteId, title FROM notes WHERE isDeleted = 0 AND isProtected = 1`);
 
     for (const noteId in protectedNoteTitles) {

src/services/options_init.js

@@ -5,21 +5,14 @@ const appInfo = require('./app_info');
 const utils = require('./utils');
 const dateUtils = require('./date_utils');
 
-async function initOptions(startNotePath, username, password) {
+async function initDocumentOptions() {
     await optionService.createOption('documentId', utils.randomSecureToken(16), false);
     await optionService.createOption('documentSecret', utils.randomSecureToken(16), false);
+}
 
-    await optionService.createOption('startNotePath', startNotePath, false);
+async function initSyncedOptions(username, password) {
-    await optionService.createOption('protectedSessionTimeout', 600, true);
+    await optionService.createOption('protectedSessionTimeout', 600);
-    await optionService.createOption('noteRevisionSnapshotTimeInterval', 600, true);
+    await optionService.createOption('noteRevisionSnapshotTimeInterval', 600);
-    await optionService.createOption('lastBackupDate', dateUtils.nowDate(), false);
-    await optionService.createOption('dbVersion', appInfo.dbVersion, false);
-
-    await optionService.createOption('lastSyncedPull', appInfo.dbVersion, false);
-    await optionService.createOption('lastSyncedPush', 0, false);
-
-    await optionService.createOption('zoomFactor', 1.0, false);
-    await optionService.createOption('theme', 'white', false);
 
     await optionService.createOption('username', username);
 
@@ -34,12 +27,26 @@ async function initOptions(startNotePath, username, password) {
     await optionService.createOption('encryptedDataKeyIv', '');
 
     await passwordEncryptionService.setDataKey(password, utils.randomSecureToken(16));
+}
 
-    await optionService.createOption('syncServerHost', '', false);
+async function initNotSyncedOptions(startNotePath = '', syncServerHost = '') {
+    await optionService.createOption('startNotePath', startNotePath, false);
+    await optionService.createOption('lastBackupDate', dateUtils.nowDate(), false);
+    await optionService.createOption('dbVersion', appInfo.dbVersion, false);
+
+    await optionService.createOption('lastSyncedPull', appInfo.dbVersion, false);
+    await optionService.createOption('lastSyncedPush', 0, false);
+
+    await optionService.createOption('zoomFactor', 1.0, false);
+    await optionService.createOption('theme', 'white', false);
+
+    await optionService.createOption('syncServerHost', syncServerHost, false);
     await optionService.createOption('syncServerTimeout', 5000, false);
     await optionService.createOption('syncProxy', '', false);
 }
 
 module.exports = {
-    initOptions
+    initDocumentOptions,
+    initSyncedOptions,
+    initNotSyncedOptions
 };

src/services/sql_init.js

@@ -12,9 +12,13 @@ async function createConnection() {
 }
 
 let dbReadyResolve = null;
-const dbReady = new Promise((resolve, reject) => {
+const dbReady = new Promise(async (resolve, reject) => {
     dbReadyResolve = resolve;
 
+    // no need to create new connection now since DB stays the same all the time
+    const db = await createConnection();
+    sql.setDbConnection(db);
+
     initDbConnection();
 });
 
@@ -26,9 +30,6 @@ async function isDbInitialized() {
 
 async function initDbConnection() {
     await cls.init(async () => {
-        const db = await createConnection();
-        sql.setDbConnection(db);
-
         await sql.execute("PRAGMA foreign_keys = ON");
 
         if (!await isDbInitialized()) {
@@ -45,12 +46,12 @@ async function initDbConnection() {
         }
 
         log.info("DB ready.");
-        dbReadyResolve(db);
+        dbReadyResolve();
     });
 }
 
 async function createInitialDatabase(username, password) {
-    log.info("Connected to db, but schema doesn't exist. Initializing schema ...");
+    log.info("Creating initial database ...");
 
     const schema = fs.readFileSync(resourceDir.DB_INIT_DIR + '/schema.sql', 'UTF-8');
     const notesSql = fs.readFileSync(resourceDir.DB_INIT_DIR + '/main_notes.sql', 'UTF-8');
@@ -67,15 +68,34 @@ async function createInitialDatabase(username, password) {
 
         const startNoteId = await sql.getValue("SELECT noteId FROM branches WHERE parentNoteId = 'root' AND isDeleted = 0 ORDER BY notePosition");
 
-        await require('./options_init').initOptions(startNoteId, username, password);
+        const optionsInitService = require('./options_init');
+
+        await optionsInitService.initDocumentOptions();
+        await optionsInitService.initSyncedOptions(username, password);
+        await optionsInitService.initNotSyncedOptions(startNoteId);
+
         await require('./sync_table').fillAllSyncRows();
     });
 
-    log.info("Schema and initial content generated. Waiting for user to enter username/password to finish setup.");
+    log.info("Schema and initial content generated.");
 
     await initDbConnection();
 }
 
+async function createDatabaseForSync(syncServerHost) {
+    log.info("Creating database for sync with server ...");
+
+    const schema = fs.readFileSync(resourceDir.DB_INIT_DIR + '/schema.sql', 'UTF-8');
+
+    await sql.transactional(async () => {
+        await sql.executeScript(schema);
+
+        await require('./options_init').initNotSyncedOptions('', syncServerHost);
+    });
+
+    log.info("Schema and not synced options generated.");
+}
+
 async function isDbUpToDate() {
     const dbVersion = parseInt(await sql.getValue("SELECT value FROM options WHERE name = 'dbVersion'"));
 
@@ -93,5 +113,6 @@ module.exports = {
     isDbInitialized,
     initDbConnection,
     isDbUpToDate,
-    createInitialDatabase
+    createInitialDatabase,
+    createDatabaseForSync
 };

src/services/sync.js

@@ -18,6 +18,11 @@ const cls = require('./cls');
 
 let proxyToggle = true;
 
+const stats = {
+    outstandingPushes: 0,
+    outstandingPulls: 0
+};
+
 async function sync() {
     try {
         await syncMutexService.doExclusively(async () => {
@@ -82,21 +87,33 @@ async function login() {
 }
 
 async function pullSync(syncContext) {
-    const changesUri = '/api/sync/changed?lastSyncId=' + await getLastSyncedPull();
+    while (true) {
+        const lastSyncedPull = await getLastSyncedPull();
+        const changesUri = '/api/sync/changed?lastSyncId=' + lastSyncedPull;
 
-    const rows = await syncRequest(syncContext, 'GET', changesUri);
+        const resp = await syncRequest(syncContext, 'GET', changesUri);
+        stats.outstandingPulls = resp.maxSyncId - lastSyncedPull;
 
-    log.info("Pulled " + rows.length + " changes from " + changesUri);
+        const rows = resp.syncs;
 
-    for (const {sync, entity} of rows) {
+        if (rows.length === 0) {
-        if (sourceIdService.isLocalSourceId(sync.sourceId)) {
+            break;
-            log.info(`Skipping pull #${sync.id} ${sync.entityName} ${sync.entityId} because ${sync.sourceId} is a local source id.`);
-        }
-        else {
-            await syncUpdateService.updateEntity(sync, entity, syncContext.sourceId);
         }
 
-        await setLastSyncedPull(sync.id);
+        log.info("Pulled " + rows.length + " changes from " + changesUri);
+
+        for (const {sync, entity} of rows) {
+            if (sourceIdService.isLocalSourceId(sync.sourceId)) {
+                log.info(`Skipping pull #${sync.id} ${sync.entityName} ${sync.entityId} because ${sync.sourceId} is a local source id.`);
+            }
+            else {
+                await syncUpdateService.updateEntity(sync, entity, syncContext.sourceId);
+            }
+
+            stats.outstandingPulls = resp.maxSyncId - sync.id;
+
+            await setLastSyncedPull(sync.id);
+        }
     }
 
     log.info("Finished pull");
@@ -127,6 +144,8 @@ async function pushSync(syncContext) {
         if (filteredSyncs.length === 0) {
             log.info("Nothing to push");
 
+            stats.outstandingPushes = 0;
+
             await setLastSyncedPush(lastSyncedPush);
 
             break;
@@ -144,6 +163,8 @@ async function pushSync(syncContext) {
         lastSyncedPush = syncRecords[syncRecords.length - 1].sync.id;
 
         await setLastSyncedPush(lastSyncedPush);
+
+        stats.outstandingPushes = await sql.getValue(`SELECT MAX(id) FROM sync`) - lastSyncedPush;
     }
 }
 
@@ -290,5 +311,6 @@ sqlInit.dbReady.then(async () => {
 module.exports = {
     sync,
     login,
-    getSyncRecords
+    getSyncRecords,
+    stats
 };

src/views/setup.ejs

@@ -86,6 +86,16 @@
 
         <button type="button" data-bind="click: finish" class="btn btn-primary">Finish setup</button>
     </div>
+
+    <div data-bind="visible: step() == 'sync-in-progress'">
+        <h2>Sync in progress</h2>
+
+        <div class="alert alert-success">Sync has been correctly set up. It will take some time for the initial sync to finish. Once it's done, you'll be redirected to the login page.</div>
+
+        <div>
+            Outstanding sync items: <strong id="outstanding-syncs">N/A</strong>
+        </div>
+    </div>
 </div>
 
 <script type="text/javascript">