image sync

package-lock.json

@@ -4167,6 +4167,34 @@
         "es5-ext": "0.10.35"
       }
     },
+    "exec-buffer": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/exec-buffer/-/exec-buffer-3.2.0.tgz",
+      "integrity": "sha512-wsiD+2Tp6BWHoVv3B+5Dcx6E7u5zky+hUwOHjuH2hKSLR3dvRmX8fk8UD8uqQixHs4Wk6eDmiegVrMPjKj7wpA==",
+      "requires": {
+        "execa": "0.7.0",
+        "p-finally": "1.0.0",
+        "pify": "3.0.0",
+        "rimraf": "2.6.2",
+        "tempfile": "2.0.0"
+      },
+      "dependencies": {
+        "pify": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz",
+          "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY="
+        },
+        "tempfile": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/tempfile/-/tempfile-2.0.0.tgz",
+          "integrity": "sha1-awRGhWqbERTRhW/8vlCczLCXcmU=",
+          "requires": {
+            "temp-dir": "1.0.0",
+            "uuid": "3.1.0"
+          }
+        }
+      }
+    },
     "exec-series": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/exec-series/-/exec-series-1.0.3.tgz",
@@ -4180,7 +4208,6 @@
       "version": "0.7.0",
       "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz",
       "integrity": "sha1-lEvs00zEHuMqY6n68nrVpl/Fl3c=",
-      "dev": true,
       "requires": {
         "cross-spawn": "5.1.0",
         "get-stream": "3.0.0",
@@ -5461,6 +5488,16 @@
         }
       }
     },
+    "imagemin-pngquant": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/imagemin-pngquant/-/imagemin-pngquant-5.0.1.tgz",
+      "integrity": "sha1-2KMp2lU6+iJrEc5i3r4Lfje0OeY=",
+      "requires": {
+        "exec-buffer": "3.2.0",
+        "is-png": "1.1.0",
+        "pngquant-bin": "3.1.1"
+      }
+    },
     "import-lazy": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/import-lazy/-/import-lazy-2.1.0.tgz",
@@ -5826,6 +5863,11 @@
       "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=",
       "dev": true
     },
+    "is-png": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/is-png/-/is-png-1.1.0.tgz",
+      "integrity": "sha1-1XSxK/J1wDUEVVcLDltXqwYgd84="
+    },
     "is-posix-bracket": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/is-posix-bracket/-/is-posix-bracket-0.1.1.tgz",
@@ -7826,6 +7868,16 @@
       "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.3.1.tgz",
       "integrity": "sha512-ggXCTsqHRIsGMkHlCEhbHhUmNTA2r1lpkE0NL4Q9S8spkXbm4vE9TVmPso2AGYn90Gltdz8W5CyzhcIGg2Gejg=="
     },
+    "pngquant-bin": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/pngquant-bin/-/pngquant-bin-3.1.1.tgz",
+      "integrity": "sha1-0STZinWpSH9AwWQLTb/Lsr1aH9E=",
+      "requires": {
+        "bin-build": "2.2.0",
+        "bin-wrapper": "3.0.2",
+        "logalot": "2.1.0"
+      }
+    },
     "postcss": {
       "version": "5.2.18",
       "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
@@ -10309,6 +10361,11 @@
         }
       }
     },
+    "temp-dir": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-1.0.0.tgz",
+      "integrity": "sha1-CnwOom06Oa+n4OvqnB/AvE2qAR0="
+    },
     "tempfile": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/tempfile/-/tempfile-1.1.1.tgz",

package.json

@@ -35,6 +35,7 @@
     "html": "^1.0.0",
     "imagemin": "^5.3.1",
     "imagemin-mozjpeg": "^7.0.0",
+    "imagemin-pngquant": "^5.0.1",
     "ini": "^1.3.4",
     "jimp": "^0.2.28",
     "multer": "^1.3.0",

routes/api/image.js

@@ -5,6 +5,7 @@ const router = express.Router();
 const sql = require('../../services/sql');
 const auth = require('../../services/auth');
 const utils = require('../../services/utils');
+const sync_table = require('../../services/sync_table');
 const multer = require('multer')();
 const imagemin = require('imagemin');
 const imageminMozJpeg = require('imagemin-mozjpeg');
@@ -24,6 +25,7 @@ router.get('/:imageId/:filename', auth.checkApiAuth, async (req, res, next) => {
 });
 
 router.post('/upload', auth.checkApiAuth, multer.single('upload'), async (req, res, next) => {
+    const sourceId = req.headers.source_id;
     const file = req.file;
 
     const imageId = utils.newNoteId();
@@ -37,6 +39,7 @@ router.post('/upload', auth.checkApiAuth, multer.single('upload'), async (req, r
     const resizedImage = await resize(file.buffer);
     const optimizedImage = await optimize(resizedImage);
 
+    await sql.doInTransaction(async () => {
         await sql.insert("images", {
             image_id: imageId,
             format: file.mimetype.substr(6),
@@ -48,6 +51,9 @@ router.post('/upload', auth.checkApiAuth, multer.single('upload'), async (req, r
             date_created: now
         });
 
+        await sync_table.addImageSync(imageId, sourceId);
+    });
+
     res.send({
         uploaded: true,
         url: `/api/image/${imageId}/${file.originalname}`
@@ -60,8 +66,6 @@ const MAX_BYTE_SIZE = 200000; // images should have under 100 KBs
 async function resize(buffer) {
     const image = await jimp.read(buffer);
 
-    console.log("Size: ", buffer.byteLength);
-
     if (image.bitmap.width > image.bitmap.height && image.bitmap.width > MAX_SIZE) {
         image.resize(MAX_SIZE, jimp.AUTO);
     }

routes/api/sync.js

@@ -122,6 +122,17 @@ router.get('/recent_notes/:noteTreeId', auth.checkApiAuth, async (req, res, next
     res.send(await sql.getFirst("SELECT * FROM recent_notes WHERE note_tree_id = ?", [noteTreeId]));
 });
 
+router.get('/images/:imageId', auth.checkApiAuth, async (req, res, next) => {
+    const imageId = req.params.imageId;
+    const entity = await sql.getFirst("SELECT * FROM images WHERE image_id = ?", [imageId]);
+
+    if (entity && entity.data !== null) {
+        entity.data = entity.data.toString('base64');
+    }
+
+    res.send(entity);
+});
+
 router.put('/notes', auth.checkApiAuth, async (req, res, next) => {
     await syncUpdate.updateNote(req.body.entity, req.body.sourceId);
 
@@ -158,4 +169,10 @@ router.put('/recent_notes', auth.checkApiAuth, async (req, res, next) => {
     res.send({});
 });
 
+router.put('/images', auth.checkApiAuth, async (req, res, next) => {
+    await syncUpdate.updateImage(req.body.entity, req.body.sourceId);
+
+    res.send({});
+});
+
 module.exports = router;

services/consistency_checks.js

@@ -4,8 +4,11 @@ const sql = require('./sql');
 const log = require('./log');
 const messaging = require('./messaging');
 const sync_mutex = require('./sync_mutex');
+const utils = require('./utils');
 
 async function runCheck(query, errorText, errorList) {
+    utils.assertArguments(query, errorText, errorList);
+
     const result = await sql.getFirstColumn(query);
 
     if (result.length > 0) {
@@ -138,7 +141,7 @@ async function runAllChecks() {
           WHERE
             (SELECT COUNT(*) FROM notes_tree WHERE notes.note_id = notes_tree.note_id AND notes_tree.is_deleted = 0) = 0
             AND notes.is_deleted = 0
-    `,);
+    `, 'No undeleted note trees for note IDs', errorList);
 
     await runCheck(`
           SELECT 

services/content_hash.js

@@ -19,7 +19,8 @@ async function getHashes() {
     const optionsQuestionMarks = Array(options.SYNCED_OPTIONS.length).fill('?').join(',');
 
     const hashes = {
-        notes: getHash(await sql.getAll(`SELECT
+        notes: getHash(await sql.getAll(`
+            SELECT
               note_id,
               note_title,
               note_text,
@@ -29,7 +30,8 @@ async function getHashes() {
             FROM notes
             ORDER BY note_id`)),
 
-        notes_tree: getHash(await sql.getAll(`SELECT
+        notes_tree: getHash(await sql.getAll(`
+            SELECT
                note_tree_id,
                note_id,
                parent_note_id,
@@ -40,7 +42,8 @@ async function getHashes() {
              FROM notes_tree
              ORDER BY note_tree_id`)),
 
-        notes_history: getHash(await sql.getAll(`SELECT
+        notes_history: getHash(await sql.getAll(`
+            SELECT
               note_history_id,
               note_id,
               note_title,
@@ -50,7 +53,8 @@ async function getHashes() {
             FROM notes_history
             ORDER BY note_history_id`)),
 
-        recent_notes: getHash(await sql.getAll(`SELECT
+        recent_notes: getHash(await sql.getAll(`
+           SELECT
              note_tree_id,
              note_path,
              date_accessed,
@@ -58,12 +62,27 @@ async function getHashes() {
            FROM recent_notes
            ORDER BY note_path`)),
 
-        options: getHash(await sql.getAll(`SELECT 
+        options: getHash(await sql.getAll(`
+           SELECT 
              opt_name,
              opt_value 
            FROM options 
            WHERE opt_name IN (${optionsQuestionMarks}) 
-                                                  ORDER BY opt_name`, options.SYNCED_OPTIONS))
+           ORDER BY opt_name`, options.SYNCED_OPTIONS)),
+
+        // we don't include image data on purpose because they are quite large, checksum is good enough
+        // to represent the data anyway
+        images: getHash(await sql.getAll(`
+          SELECT 
+            image_id,
+            format,
+            checksum,
+            name,
+            is_deleted,
+            date_modified,
+            date_created
+          FROM images  
+          ORDER BY image_id`))
     };
 
     const elapseTimeMs = new Date().getTime() - startTime.getTime();

services/sync.js

@@ -143,6 +143,9 @@ async function pullSync(syncContext) {
         else if (sync.entity_name === 'recent_notes') {
             await syncUpdate.updateRecentNotes(resp, syncContext.sourceId);
         }
+        else if (sync.entity_name === 'images') {
+            await syncUpdate.updateImage(resp, syncContext.sourceId);
+        }
         else {
             throw new Error(`Unrecognized entity type ${sync.entity_name} in sync #${sync.id}`);
         }
@@ -214,6 +217,13 @@ async function pushEntity(sync, syncContext) {
     else if (sync.entity_name === 'recent_notes') {
         entity = await sql.getFirst('SELECT * FROM recent_notes WHERE note_tree_id = ?', [sync.entity_id]);
     }
+    else if (sync.entity_name === 'images') {
+        entity = await sql.getFirst('SELECT * FROM images WHERE image_id = ?', [sync.entity_id]);
+
+        if (entity.data !== null) {
+            entity.data = entity.data.toString('base64');
+        }
+    }
     else {
         throw new Error(`Unrecognized entity type ${sync.entity_name} in sync #${sync.id}`);
     }

services/sync_table.js

@@ -28,6 +28,10 @@ async function addRecentNoteSync(noteTreeId, sourceId) {
     await addEntitySync("recent_notes", noteTreeId, sourceId);
 }
 
+async function addImageSync(imageId, sourceId) {
+    await addEntitySync("images", imageId, sourceId);
+}
+
 async function addEntitySync(entityName, entityId, sourceId) {
     await sql.replace("sync", {
         entity_name: entityName,
@@ -78,6 +82,7 @@ async function fillAllSyncRows() {
     await fillSyncRows("notes_tree", "note_tree_id");
     await fillSyncRows("notes_history", "note_history_id");
     await fillSyncRows("recent_notes", "note_tree_id");
+    await fillSyncRows("images", "image_id");
 }
 
 module.exports = {
@@ -87,6 +92,7 @@ module.exports = {
     addNoteHistorySync,
     addOptionsSync,
     addRecentNoteSync,
+    addImageSync,
     cleanupSyncRowsForMissingEntities,
     fillAllSyncRows
 };

services/sync_update.js

@@ -92,11 +92,30 @@ async function updateRecentNotes(entity, sourceId) {
     }
 }
 
+async function updateImage(entity, sourceId) {
+    if (entity.data !== null) {
+        entity.data = Buffer.from(entity.data, 'base64');
+    }
+
+    const origImage = await sql.getFirst("SELECT * FROM images WHERE image_id = ?", [entity.image_id]);
+
+    if (!origImage || origImage.date_modified <= entity.date_modified) {
+        await sql.doInTransaction(async () => {
+            await sql.replace("images", entity);
+
+            await sync_table.addImageSync(entity.image_id, sourceId);
+        });
+
+        log.info("Update/sync image " + entity.image_id);
+    }
+}
+
 module.exports = {
     updateNote,
     updateNoteTree,
     updateNoteHistory,
     updateNoteReordering,
     updateOptions,
-    updateRecentNotes
+    updateRecentNotes,
+    updateImage
 };

services/utils.js

@@ -79,6 +79,14 @@ function sanitizeSql(str) {
     return str.replace(/'/g, "\\'");
 }
 
+function assertArguments() {
+    for (const i in arguments) {
+        if (!arguments[i]) {
+            throw new Error(`Argument idx#${i} should not be falsy: ${arguments[i]}`);
+        }
+    }
+}
+
 module.exports = {
     randomSecureToken,
     randomString,
@@ -95,5 +103,6 @@ module.exports = {
     hash,
     isEmptyOrWhitespace,
     getDateTimeForFile,
-    sanitizeSql
+    sanitizeSql,
+    assertArguments
 };