Tools/Notes
1
0
Fork 0
mirror of https://github.com/TriliumNext/Notes.git synced 2025-11-04 15:11:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactored "sync" table to "entity_changes"

Browse Source
This commit is contained in:
zadam 2020-08-02 23:27:48 +02:00
parent 5a5ea4f474
commit 864271d5ef
32 changed files with 170 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
db/migrations/0104__fill_sync_rows_for_options.js
View File

@ -1,4 +1,4 @@
const syncTableService = require('../../src/services/sync_table');
const syncTableService = require('../../src/services/entity_changes.js');
// options has not been filled so far which caused problems with clean-slate sync.
module.exports = async () => await syncTableService.fillAllSyncRows();

2
db/migrations/0132__fill_sync.js
View File

@ -1,4 +1,4 @@
const syncTableService = require('../../src/services/sync_table');
const syncTableService = require('../../src/services/entity_changes.js');
module.exports = async () => {
await syncTableService.fillAllSyncRows();

1
db/migrations/0163__rename_sync_to_entity_changes.sql Normal file
View File

@ -0,0 +1 @@
ALTER TABLE sync RENAME TO entity_changes;

5
src/entities/attribute.js
View File

@ -102,11 +102,6 @@ class Attribute extends Entity {
}
}
// cannot be static!
updatePojo(pojo) {
delete pojo.__note; // FIXME: probably note necessary anymore
}
createClone(type, name, value, isInheritable) {
return new Attribute({
noteId: this.noteId,

4
src/entities/note.js
View File

@ -6,7 +6,7 @@ const protectedSessionService = require('../services/protected_session');
const sql = require('../services/sql');
const utils = require('../services/utils');
const dateUtils = require('../services/date_utils');
const syncTableService = require('../services/sync_table');
const entityChangesService = require('../services/entity_changes.js');
const LABEL = 'label';
const LABEL_DEFINITION = 'label-definition';
@ -154,7 +154,7 @@ class Note extends Entity {
sql.upsert("note_contents", "noteId", pojo);
syncTableService.addNoteContentSync(this.noteId);
entityChangesService.addNoteContentSync(this.noteId);
}
setJsonContent(content) {

4
src/entities/note_revision.js
View File

@ -5,7 +5,7 @@ const protectedSessionService = require('../services/protected_session');
const utils = require('../services/utils');
const sql = require('../services/sql');
const dateUtils = require('../services/date_utils');
const syncTableService = require('../services/sync_table');
const entityChangesService = require('../services/entity_changes.js');
/**
* NoteRevision represents snapshot of note's title and content at some point in the past. It's used for seamless note versioning.
@ -126,7 +126,7 @@ class NoteRevision extends Entity {
sql.upsert("note_revision_contents", "noteRevisionId", pojo);
syncTableService.addNoteRevisionContentSync(this.noteRevisionId);
entityChangesService.addNoteRevisionContentSync(this.noteRevisionId);
}
beforeSaving() {

4
src/public/app/dialogs/options/advanced.js
View File

@ -41,7 +41,7 @@ export default class AdvancedOptions {
$("#options-advanced").html(TPL);
this.$forceFullSyncButton = $("#force-full-sync-button");
this.$fillSyncRowsButton = $("#fill-sync-rows-button");
this.$fillEntityChangesButton = $("#fill-sync-rows-button");
this.$anonymizeButton = $("#anonymize-button");
this.$backupDatabaseButton = $("#backup-database-button");
this.$vacuumDatabaseButton = $("#vacuum-database-button");
@ -53,7 +53,7 @@ export default class AdvancedOptions {
toastService.showMessage("Full sync triggered");
});
this.$fillSyncRowsButton.on('click', async () => {
this.$fillEntityChangesButton.on('click', async () => {
await server.post('sync/fill-sync-rows');
toastService.showMessage("Sync rows filled successfully");

2
src/public/app/services/entrypoints.js
View File

@ -72,7 +72,7 @@ export default class Entrypoints extends Component {
isProtected: todayNote.isProtected
});
await ws.waitForMaxKnownSyncId();
await ws.waitForMaxKnownEntityChangeId();
await appContext.tabManager.openTabWithNote(note.noteId, true);

6
src/public/app/services/frontend_script_api.js
View File

@ -67,7 +67,7 @@ function FrontendScriptApi(startNote, currentNote, originEntity = null, $contain
* @return {Promise<void>}
*/
this.activateNewNote = async notePath => {
await ws.waitForMaxKnownSyncId();
await ws.waitForMaxKnownEntityChangeId();
await appContext.tabManager.getActiveTabContext().setNote(notePath);
appContext.triggerEvent('focusAndSelectTitle');
@ -152,7 +152,7 @@ function FrontendScriptApi(startNote, currentNote, originEntity = null, $contain
if (ret.success) {
// wait until all the changes done in the script has been synced to frontend before continuing
await ws.waitForSyncId(ret.maxSyncId);
await ws.waitForEntityChangeId(ret.maxEntityChangeId);
return ret.executionResult;
}
@ -402,7 +402,7 @@ function FrontendScriptApi(startNote, currentNote, originEntity = null, $contain
*
* @method
*/
this.waitUntilSynced = ws.waitForMaxKnownSyncId;
this.waitUntilSynced = ws.waitForMaxKnownEntityChangeId;
/**
* This will refresh all currently opened notes which have included note specified in the parameter

4
src/public/app/services/note_create.js
View File

@ -49,7 +49,7 @@ async function createNote(parentNoteId, options = {}) {
window.cutToNote.removeSelection();
}
await ws.waitForMaxKnownSyncId();
await ws.waitForMaxKnownEntityChangeId();
if (options.activate) {
const activeTabContext = appContext.tabManager.getActiveTabContext();
@ -91,7 +91,7 @@ function parseSelectedHtml(selectedHtml) {
async function duplicateNote(noteId, parentNoteId) {
const {note} = await server.post(`notes/${noteId}/duplicate/${parentNoteId}`);
await ws.waitForMaxKnownSyncId();
await ws.waitForMaxKnownEntityChangeId();
await appContext.tabManager.activateOrOpenNote(note.noteId);

10
src/public/app/services/server.js
View File

@ -45,7 +45,7 @@ async function remove(url, sourceId) {
let i = 1;
const reqResolves = {};
let maxKnownSyncId = 0;
let maxKnownEntityChangeId = 0;
async function call(method, url, data, headers = {}) {
let resp;
@ -82,10 +82,10 @@ async function call(method, url, data, headers = {}) {
console.log(`${method} ${url} took ${end - start}ms`);
}
const maxSyncIdStr = resp.headers['trilium-max-sync-id'];
const maxEntityChangeIdStr = resp.headers['trilium-max-entity-change-id'];
if (maxSyncIdStr && maxSyncIdStr.trim()) {
maxKnownSyncId = Math.max(maxKnownSyncId, parseInt(maxSyncIdStr));
if (maxEntityChangeIdStr && maxEntityChangeIdStr.trim()) {
maxKnownEntityChangeId = Math.max(maxKnownEntityChangeId, parseInt(maxEntityChangeIdStr));
}
return resp.body;
@ -160,5 +160,5 @@ export default {
ajax,
// don't remove, used from CKEditor image upload!
getHeaders,
getMaxKnownSyncId: () => maxKnownSyncId
getMaxKnownEntityChangeId: () => maxKnownEntityChangeId
};

65
src/public/app/services/ws.js
View File

@ -13,8 +13,8 @@ const $outstandingSyncsCount = $("#outstanding-syncs-count");
const messageHandlers = [];
let ws;
let lastAcceptedSyncId = window.glob.maxSyncIdAtLoad;
let lastProcessedSyncId = window.glob.maxSyncIdAtLoad;
let lastAcceptedEntityChangeId = window.glob.maxEntityChangeIdAtLoad;
let lastProcessedEntityChangeId = window.glob.maxEntityChangeIdAtLoad;
let lastPingTs;
let syncDataQueue = [];
@ -38,13 +38,12 @@ function subscribeToMessages(messageHandler) {
// used to serialize sync operations
let consumeQueuePromise = null;
// most sync events are sent twice - once immediatelly after finishing the transaction and once during the scheduled ping
// but we want to process only once
const processedSyncIds = new Set();
// to make sure each change event is processed only once. Not clear if this is still necessary
const processedEntityChangeIds = new Set();
function logRows(syncRows) {
const filteredRows = syncRows.filter(row =>
!processedSyncIds.has(row.id)
!processedEntityChangeIds.has(row.id)
&& row.entityName !== 'recent_notes'
&& (row.entityName !== 'options' || row.entityId !== 'openTabs'));
@ -71,8 +70,8 @@ async function handleMessage(event) {
syncDataQueue.push(...syncRows);
// we set lastAcceptedSyncId even before sync processing and send ping so that backend can start sending more updates
lastAcceptedSyncId = Math.max(lastAcceptedSyncId, syncRows[syncRows.length - 1].id);
// we set lastAcceptedEntityChangeId even before sync processing and send ping so that backend can start sending more updates
lastAcceptedEntityChangeId = Math.max(lastAcceptedEntityChangeId, syncRows[syncRows.length - 1].id);
sendPing();
// first wait for all the preceding consumers to finish
@ -100,38 +99,38 @@ async function handleMessage(event) {
}
}
let syncIdReachedListeners = [];
let entityChangeIdReachedListeners = [];
function waitForSyncId(desiredSyncId) {
if (desiredSyncId <= lastProcessedSyncId) {
function waitForEntityChangeId(desiredEntityChangeId) {
if (desiredEntityChangeId <= lastProcessedEntityChangeId) {
return Promise.resolve();
}
console.debug("Waiting for", desiredSyncId, 'current is', lastProcessedSyncId);
console.debug("Waiting for", desiredEntityChangeId, 'current is', lastProcessedEntityChangeId);
return new Promise((res, rej) => {
syncIdReachedListeners.push({
desiredSyncId,
entityChangeIdReachedListeners.push({
desiredEntityChangeId: desiredEntityChangeId,
resolvePromise: res,
start: Date.now()
})
});
}
function waitForMaxKnownSyncId() {
return waitForSyncId(server.getMaxKnownSyncId());
function waitForMaxKnownEntityChangeId() {
return waitForEntityChangeId(server.getMaxKnownEntityChangeId());
}
function checkSyncIdListeners() {
syncIdReachedListeners
.filter(l => l.desiredSyncId <= lastProcessedSyncId)
function checkEntityChangeIdListeners() {
entityChangeIdReachedListeners
.filter(l => l.desiredEntityChangeId <= lastProcessedEntityChangeId)
.forEach(l => l.resolvePromise());
syncIdReachedListeners = syncIdReachedListeners
.filter(l => l.desiredSyncId > lastProcessedSyncId);
entityChangeIdReachedListeners = entityChangeIdReachedListeners
.filter(l => l.desiredEntityChangeId > lastProcessedEntityChangeId);
syncIdReachedListeners.filter(l => Date.now() > l.start - 60000)
.forEach(l => console.log(`Waiting for syncId ${l.desiredSyncId} while current is ${lastProcessedSyncId} for ${Math.floor((Date.now() - l.start) / 1000)}s`));
entityChangeIdReachedListeners.filter(l => Date.now() > l.start - 60000)
.forEach(l => console.log(`Waiting for entityChangeId ${l.desiredEntityChangeId} while current is ${lastProcessedEntityChangeId} for ${Math.floor((Date.now() - l.start) / 1000)}s`));
}
async function runSafely(syncHandler, syncData) {
@ -143,18 +142,12 @@ async function runSafely(syncHandler, syncData) {
}
}
/**
* TODO: we should rethink the fact that each sync row is sent twice (once at the end of transaction, once periodically)
* and we keep both lastProcessedSyncId and processedSyncIds
* it even seems incorrect that when transaction sync rows are received, we incorrectly increase lastProcessedSyncId
* and then some syncs might lost (or are *all* sync rows sent from transactions?)
*/
async function consumeSyncData() {
if (syncDataQueue.length > 0) {
const allSyncRows = syncDataQueue;
syncDataQueue = [];
const nonProcessedSyncRows = allSyncRows.filter(sync => !processedSyncIds.has(sync.id));
const nonProcessedSyncRows = allSyncRows.filter(sync => !processedEntityChangeIds.has(sync.id));
try {
await utils.timeLimit(processSyncRows(nonProcessedSyncRows), 5000);
@ -167,13 +160,13 @@ async function consumeSyncData() {
}
for (const syncRow of nonProcessedSyncRows) {
processedSyncIds.add(syncRow.id);
processedEntityChangeIds.add(syncRow.id);
}
lastProcessedSyncId = Math.max(lastProcessedSyncId, allSyncRows[allSyncRows.length - 1].id);
lastProcessedEntityChangeId = Math.max(lastProcessedEntityChangeId, allSyncRows[allSyncRows.length - 1].id);
}
checkSyncIdListeners();
checkEntityChangeIdListeners();
}
function connectWebSocket() {
@ -198,7 +191,7 @@ async function sendPing() {
if (ws.readyState === ws.OPEN) {
ws.send(JSON.stringify({
type: 'ping',
lastSyncId: lastAcceptedSyncId
lastEntityChangeId: lastAcceptedEntityChangeId
}));
}
else if (ws.readyState === ws.CLOSED || ws.readyState === ws.CLOSING) {
@ -394,6 +387,6 @@ async function processSyncRows(syncRows) {
export default {
logError,
subscribeToMessages,
waitForSyncId,
waitForMaxKnownSyncId
waitForEntityChangeId,
waitForMaxKnownEntityChangeId
};

6
src/routes/api/branches.js
View File

@ -2,7 +2,7 @@
const sql = require('../../services/sql');
const utils = require('../../services/utils');
const syncTableService = require('../../services/sync_table');
const entityChangesService = require('../../services/entity_changes.js');
const treeService = require('../../services/tree');
const noteService = require('../../services/notes');
const repository = require('../../services/repository');
@ -66,7 +66,7 @@ function moveBranchBeforeNote(req) {
sql.execute("UPDATE branches SET notePosition = notePosition + 10 WHERE parentNoteId = ? AND notePosition >= ? AND isDeleted = 0",
[beforeNote.parentNoteId, beforeNote.notePosition]);
syncTableService.addNoteReorderingSync(beforeNote.parentNoteId);
entityChangesService.addNoteReorderingSync(beforeNote.parentNoteId);
if (branchToMove.parentNoteId === beforeNote.parentNoteId) {
branchToMove.notePosition = beforeNote.notePosition;
@ -100,7 +100,7 @@ function moveBranchAfterNote(req) {
sql.execute("UPDATE branches SET notePosition = notePosition + 10 WHERE parentNoteId = ? AND notePosition > ? AND isDeleted = 0",
[afterNote.parentNoteId, afterNote.notePosition]);
syncTableService.addNoteReorderingSync(afterNote.parentNoteId);
entityChangesService.addNoteReorderingSync(afterNote.parentNoteId);
const movedNotePosition = afterNote.notePosition + 10;

2
src/routes/api/login.js
View File

@ -49,7 +49,7 @@ function loginSync(req) {
return {
sourceId: sourceIdService.getCurrentSourceId(),
maxSyncId: sql.getValue("SELECT COALESCE(MAX(id), 0) FROM sync WHERE isSynced = 1")
maxEntityChangeId: sql.getValue("SELECT COALESCE(MAX(id), 0) FROM entity_changes WHERE isSynced = 1")
};
}

2
src/routes/api/script.js
View File

@ -13,7 +13,7 @@ function exec(req) {
return {
success: true,
executionResult: result,
maxSyncId: syncService.getMaxSyncId()
maxEntityChangeId: syncService.getMaxEntityChangeId()
};
}
catch (e) {

36
src/routes/api/sync.js
View File

@ -2,7 +2,7 @@
const syncService = require('../../services/sync');
const syncUpdateService = require('../../services/sync_update');
const syncTableService = require('../../services/sync_table');
const entityChangesService = require('../../services/entity_changes.js');
const sql = require('../../services/sql');
const sqlInit = require('../../services/sql_init');
const optionService = require('../../services/options');
@ -50,7 +50,7 @@ function getStats() {
function checkSync() {
return {
entityHashes: contentHashService.getEntityHashes(),
maxSyncId: sql.getValue('SELECT COALESCE(MAX(id), 0) FROM sync WHERE isSynced = 1')
maxEntityChangeId: sql.getValue('SELECT COALESCE(MAX(id), 0) FROM entity_changes WHERE isSynced = 1')
};
}
@ -60,8 +60,8 @@ function syncNow() {
return syncService.sync();
}
function fillSyncRows() {
syncTableService.fillAllSyncRows();
function fillEntityChanges() {
entityChangesService.fillAllEntityChanges();
log.info("Sync rows have been filled.");
}
@ -82,32 +82,32 @@ function forceNoteSync(req) {
const now = dateUtils.utcNowDateTime();
sql.execute(`UPDATE notes SET utcDateModified = ? WHERE noteId = ?`, [now, noteId]);
syncTableService.addNoteSync(noteId);
entityChangesService.addNoteSync(noteId);
sql.execute(`UPDATE note_contents SET utcDateModified = ? WHERE noteId = ?`, [now, noteId]);
syncTableService.addNoteContentSync(noteId);
entityChangesService.addNoteContentSync(noteId);
for (const branchId of sql.getColumn("SELECT branchId FROM branches WHERE noteId = ?", [noteId])) {
sql.execute(`UPDATE branches SET utcDateModified = ? WHERE branchId = ?`, [now, branchId]);
syncTableService.addBranchSync(branchId);
entityChangesService.addBranchSync(branchId);
}
for (const attributeId of sql.getColumn("SELECT attributeId FROM attributes WHERE noteId = ?", [noteId])) {
sql.execute(`UPDATE attributes SET utcDateModified = ? WHERE attributeId = ?`, [now, attributeId]);
syncTableService.addAttributeSync(attributeId);
entityChangesService.addAttributeSync(attributeId);
}
for (const noteRevisionId of sql.getColumn("SELECT noteRevisionId FROM note_revisions WHERE noteId = ?", [noteId])) {
sql.execute(`UPDATE note_revisions SET utcDateModified = ? WHERE noteRevisionId = ?`, [now, noteRevisionId]);
syncTableService.addNoteRevisionSync(noteRevisionId);
entityChangesService.addNoteRevisionSync(noteRevisionId);
sql.execute(`UPDATE note_revision_contents SET utcDateModified = ? WHERE noteRevisionId = ?`, [now, noteRevisionId]);
syncTableService.addNoteRevisionContentSync(noteRevisionId);
entityChangesService.addNoteRevisionContentSync(noteRevisionId);
}
syncTableService.addRecentNoteSync(noteId);
entityChangesService.addRecentNoteSync(noteId);
log.info("Forcing note sync for " + noteId);
@ -118,17 +118,17 @@ function forceNoteSync(req) {
function getChanged(req) {
const startTime = Date.now();
const lastSyncId = parseInt(req.query.lastSyncId);
const lastEntityChangeId = parseInt(req.query.lastEntityChangedId);
const syncs = sql.getRows("SELECT * FROM sync WHERE isSynced = 1 AND id > ? LIMIT 1000", [lastSyncId]);
const entityChanges = sql.getRows("SELECT * FROM entity_changes WHERE isSynced = 1 AND id > ? LIMIT 1000", [lastEntityChangeId]);
const ret = {
syncs: syncService.getSyncRecords(syncs),
maxSyncId: sql.getValue('SELECT COALESCE(MAX(id), 0) FROM sync WHERE isSynced = 1')
syncs: syncService.getEntityChangesRecords(entityChanges),
maxEntityChangeId: sql.getValue('SELECT COALESCE(MAX(id), 0) FROM entity_changes WHERE isSynced = 1')
};
if (ret.syncs.length > 0) {
log.info(`Returning ${ret.syncs.length} sync records in ${Date.now() - startTime}ms`);
log.info(`Returning ${ret.syncs.length} entity changes in ${Date.now() - startTime}ms`);
}
return ret;
@ -155,14 +155,14 @@ function queueSector(req) {
const entityPrimaryKey = entityConstructor.getEntityFromEntityName(entityName).primaryKeyName;
syncTableService.addEntitySyncsForSector(entityName, entityPrimaryKey, sector);
entityChangesService.addEntityChangesForSector(entityName, entityPrimaryKey, sector);
}
module.exports = {
testSync,
checkSync,
syncNow,
fillSyncRows,
fillEntityChanges,
forceFullSync,
forceNoteSync,
getChanged,

2
src/routes/index.js
View File

@ -23,7 +23,7 @@ function index(req, res) {
treeFontSize: parseInt(options.treeFontSize),
detailFontSize: parseInt(options.detailFontSize),
sourceId: sourceIdService.generateSourceId(),
maxSyncIdAtLoad: sql.getValue("SELECT COALESCE(MAX(id), 0) FROM sync"),
maxEntityChangeIdAtLoad: sql.getValue("SELECT COALESCE(MAX(id), 0) FROM entity_changes"),
instanceName: config.General ? config.General.instanceName : null,
appCssNoteIds: getAppCssNoteIds(),
isDev: env.isDev(),

6
src/routes/routes.js
View File

@ -45,7 +45,7 @@ const auth = require('../services/auth');
const cls = require('../services/cls');
const sql = require('../services/sql');
const protectedSessionService = require('../services/protected_session');
const syncTableService = require('../services/sync_table');
const entityChangesService = require('../services/entity_changes.js');
const csurf = require('csurf');
const csrfMiddleware = csurf({
@ -54,7 +54,7 @@ const csrfMiddleware = csurf({
});
function apiResultHandler(req, res, result) {
res.setHeader('trilium-max-sync-id', syncTableService.getMaxSyncId());
res.setHeader('trilium-max-entity-change-id', entityChangesService.getMaxEntityChangeId());
// if it's an array and first element is integer then we consider this to be [statusCode, response] format
if (Array.isArray(result) && result.length > 0 && Number.isInteger(result[0])) {
@ -205,7 +205,7 @@ function register(app) {
apiRoute(POST, '/api/sync/test', syncApiRoute.testSync);
apiRoute(POST, '/api/sync/now', syncApiRoute.syncNow);
apiRoute(POST, '/api/sync/fill-sync-rows', syncApiRoute.fillSyncRows);
apiRoute(POST, '/api/sync/fill-sync-rows', syncApiRoute.fillEntityChanges);
apiRoute(POST, '/api/sync/force-full-sync', syncApiRoute.forceFullSync);
apiRoute(POST, '/api/sync/force-note-sync/:noteId', syncApiRoute.forceNoteSync);
route(GET, '/api/sync/check', [auth.checkApiAuth], syncApiRoute.checkSync, apiResultHandler);

4
src/services/app_info.js
View File

@ -4,8 +4,8 @@ const build = require('./build');
const packageJson = require('../../package');
const {TRILIUM_DATA_DIR} = require('./data_dir');
const APP_DB_VERSION = 162;
const SYNC_VERSION = 14;
const APP_DB_VERSION = 163;
const SYNC_VERSION = 15;
const CLIPPER_PROTOCOL_VERSION = "1.0";
module.exports = {

2
src/services/cloning.js
View File

@ -1,7 +1,7 @@
"use strict";
const sql = require('./sql');
const syncTable = require('./sync_table');
const syncTable = require('./entity_changes.js');
const treeService = require('./tree');
const noteService = require('./notes');
const repository = require('./repository');

8
src/services/consistency_checks.js
View File

@ -7,7 +7,7 @@ const ws = require('./ws.js');
const syncMutexService = require('./sync_mutex');
const repository = require('./repository');
const cls = require('./cls');
const syncTableService = require('./sync_table');
const entityChangesService = require('./entity_changes.js');
const optionsService = require('./options');
const Branch = require('../entities/branch');
const dateUtils = require('./date_utils');
@ -300,7 +300,7 @@ class ConsistencyChecks {
utcDateModified: dateUtils.utcNowDateTime()
});
syncTableService.addNoteContentSync(noteId);
entityChangesService.addNoteContentSync(noteId);
}
else {
// empty string might be wrong choice for some note types but it's a best guess
@ -566,7 +566,7 @@ class ConsistencyChecks {
sync.id IS NULL AND ` + (entityName === 'options' ? 'options.isSynced = 1' : '1'),
({entityId}) => {
if (this.autoFix) {
syncTableService.addEntitySync(entityName, entityId);
entityChangesService.addEntityChange(entityName, entityId);
logFix(`Created missing sync record for entityName=${entityName}, entityId=${entityId}`);
} else {
@ -585,7 +585,7 @@ class ConsistencyChecks {
AND ${key} IS NULL`,
({id, entityId}) => {
if (this.autoFix) {
sql.execute("DELETE FROM sync WHERE entityName = ? AND entityId = ?", [entityName, entityId]);
sql.execute("DELETE FROM entity_changes WHERE entityName = ? AND entityId = ?", [entityName, entityId]);
logFix(`Deleted extra sync record id=${id}, entityName=${entityName}, entityId=${entityId}`);
} else {

74
src/services/sync_table.js → src/services/entity_changes.js
View File

@ -1,16 +1,12 @@
/**
* TODO: rename "sync" table to something like "changelog" since it now also contains rows which are not synced (isSynced=false)
*/
const sql = require('./sql');
const sourceIdService = require('./source_id');
const dateUtils = require('./date_utils');
const log = require('./log');
const cls = require('./cls');
let maxSyncId = 0;
let maxEntityChangeId = 0;
function insertEntitySync(entityName, entityId, sourceId = null, isSynced = true) {
function insertEntityChange(entityName, entityId, sourceId = null, isSynced = true) {
const sync = {
entityName: entityName,
entityId: entityId,
@ -21,18 +17,18 @@ function insertEntitySync(entityName, entityId, sourceId = null, isSynced = true
sync.id = sql.replace("sync", sync);
maxSyncId = Math.max(maxSyncId, sync.id);
maxEntityChangeId = Math.max(maxEntityChangeId, sync.id);
return sync;
}
function addEntitySync(entityName, entityId, sourceId, isSynced) {
const sync = insertEntitySync(entityName, entityId, sourceId, isSynced);
function addEntityChange(entityName, entityId, sourceId, isSynced) {
const sync = insertEntityChange(entityName, entityId, sourceId, isSynced);
cls.addSyncRow(sync);
}
function addEntitySyncsForSector(entityName, entityPrimaryKey, sector) {
function addEntityChangesForSector(entityName, entityPrimaryKey, sector) {
const startTime = Date.now();
sql.transactional(() => {
@ -47,7 +43,7 @@ function addEntitySyncsForSector(entityName, entityPrimaryKey, sector) {
}
}
insertEntitySync(entityName, entityId, 'content-check', true);
insertEntityChange(entityName, entityId, 'content-check', true);
}
});
@ -57,12 +53,12 @@ function addEntitySyncsForSector(entityName, entityPrimaryKey, sector) {
function cleanupSyncRowsForMissingEntities(entityName, entityPrimaryKey) {
sql.execute(`
DELETE
FROM sync
FROM entity_changes
WHERE sync.entityName = '${entityName}'
AND sync.entityId NOT IN (SELECT ${entityPrimaryKey} FROM ${entityName})`);
}
function fillSyncRows(entityName, entityPrimaryKey, condition = '') {
function fillEntityChanges(entityName, entityPrimaryKey, condition = '') {
try {
cleanupSyncRowsForMissingEntities(entityName, entityPrimaryKey);
@ -72,7 +68,7 @@ function fillSyncRows(entityName, entityPrimaryKey, condition = '') {
let createdCount = 0;
for (const entityId of entityIds) {
const existingRows = sql.getValue("SELECT COUNT(1) FROM sync WHERE entityName = ? AND entityId = ?", [entityName, entityId]);
const existingRows = sql.getValue("SELECT COUNT(1) FROM entity_changes WHERE entityName = ? AND entityId = ?", [entityName, entityId]);
// we don't want to replace existing entities (which would effectively cause full resync)
if (existingRows === 0) {
@ -99,35 +95,35 @@ function fillSyncRows(entityName, entityPrimaryKey, condition = '') {
}
}
function fillAllSyncRows() {
function fillAllEntityChanges() {
sql.transactional(() => {
sql.execute("DELETE FROM sync");
sql.execute("DELETE FROM entity_changes");
fillSyncRows("notes", "noteId");
fillSyncRows("note_contents", "noteId");
fillSyncRows("branches", "branchId");
fillSyncRows("note_revisions", "noteRevisionId");
fillSyncRows("note_revision_contents", "noteRevisionId");
fillSyncRows("recent_notes", "noteId");
fillSyncRows("attributes", "attributeId");
fillSyncRows("api_tokens", "apiTokenId");
fillSyncRows("options", "name", 'isSynced = 1');
fillEntityChanges("notes", "noteId");
fillEntityChanges("note_contents", "noteId");
fillEntityChanges("branches", "branchId");
fillEntityChanges("note_revisions", "noteRevisionId");
fillEntityChanges("note_revision_contents", "noteRevisionId");
fillEntityChanges("recent_notes", "noteId");
fillEntityChanges("attributes", "attributeId");
fillEntityChanges("api_tokens", "apiTokenId");
fillEntityChanges("options", "name", 'isSynced = 1');
});
}
module.exports = {
addNoteSync: (noteId, sourceId) => addEntitySync("notes", noteId, sourceId),
addNoteContentSync: (noteId, sourceId) => addEntitySync("note_contents", noteId, sourceId),
addBranchSync: (branchId, sourceId) => addEntitySync("branches", branchId, sourceId),
addNoteReorderingSync: (parentNoteId, sourceId) => addEntitySync("note_reordering", parentNoteId, sourceId),
addNoteRevisionSync: (noteRevisionId, sourceId) => addEntitySync("note_revisions", noteRevisionId, sourceId),
addNoteRevisionContentSync: (noteRevisionId, sourceId) => addEntitySync("note_revision_contents", noteRevisionId, sourceId),
addOptionsSync: (name, sourceId, isSynced) => addEntitySync("options", name, sourceId, isSynced),
addRecentNoteSync: (noteId, sourceId) => addEntitySync("recent_notes", noteId, sourceId),
addAttributeSync: (attributeId, sourceId) => addEntitySync("attributes", attributeId, sourceId),
addApiTokenSync: (apiTokenId, sourceId) => addEntitySync("api_tokens", apiTokenId, sourceId),
addEntitySync,
fillAllSyncRows,
addEntitySyncsForSector,
getMaxSyncId: () => maxSyncId
addNoteSync: (noteId, sourceId) => addEntityChange("notes", noteId, sourceId),
addNoteContentSync: (noteId, sourceId) => addEntityChange("note_contents", noteId, sourceId),
addBranchSync: (branchId, sourceId) => addEntityChange("branches", branchId, sourceId),
addNoteReorderingSync: (parentNoteId, sourceId) => addEntityChange("note_reordering", parentNoteId, sourceId),
addNoteRevisionSync: (noteRevisionId, sourceId) => addEntityChange("note_revisions", noteRevisionId, sourceId),
addNoteRevisionContentSync: (noteRevisionId, sourceId) => addEntityChange("note_revision_contents", noteRevisionId, sourceId),
addOptionsSync: (name, sourceId, isSynced) => addEntityChange("options", name, sourceId, isSynced),
addRecentNoteSync: (noteId, sourceId) => addEntityChange("recent_notes", noteId, sourceId),
addAttributeSync: (attributeId, sourceId) => addEntityChange("attributes", attributeId, sourceId),
addApiTokenSync: (apiTokenId, sourceId) => addEntityChange("api_tokens", apiTokenId, sourceId),
addEntityChange,
fillAllEntityChanges,
addEntityChangesForSector,
getMaxEntityChangeId: () => maxEntityChangeId
};

2
src/services/note_cache/note_cache_loader.js
View File

@ -31,7 +31,7 @@ function load() {
}
eventService.subscribe([eventService.ENTITY_CHANGED, eventService.ENTITY_DELETED, eventService.ENTITY_SYNCED], ({entityName, entity}) => {
// note that entity can also be just POJO without methods if coming from sync
// note that entity can also be just POJO without methods if coming FROM entity_changes
if (!noteCache.loaded) {
return;

4
src/services/notes.js
View File

@ -2,7 +2,7 @@ const sql = require('./sql');
const sqlInit = require('./sql_init');
const optionService = require('./options');
const dateUtils = require('./date_utils');
const syncTableService = require('./sync_table');
const entityChangesService = require('./entity_changes.js');
const eventService = require('./events');
const repository = require('./repository');
const cls = require('../services/cls');
@ -159,7 +159,7 @@ function createNewNoteWithTarget(target, targetBranchId, params) {
const retObject = createNewNote(params);
syncTableService.addNoteReorderingSync(params.parentNoteId);
entityChangesService.addNoteReorderingSync(params.parentNoteId);
return retObject;
}

4
src/services/repository.js
View File

@ -1,7 +1,7 @@
"use strict";
const sql = require('./sql');
const syncTableService = require('../services/sync_table');
const entityChangesService = require('./entity_changes.js');
const eventService = require('./events');
const cls = require('./cls');
const entityConstructor = require('../entities/entity_constructor');
@ -119,7 +119,7 @@ function updateEntity(entity) {
if (entity.isChanged) {
const isSynced = entityName !== 'options' || entity.isSynced;
syncTableService.addEntitySync(entityName, primaryKey, null, isSynced);
entityChangesService.addEntityChange(entityName, primaryKey, null, isSynced);
if (!cls.isEntityEventsDisabled()) {
const eventPayload = {

2
src/services/setup.js
View File

@ -64,7 +64,7 @@ async function setupSyncFromSyncServer(syncServerHost, syncProxy, username, pass
}
try {
log.info("Getting document options from sync server.");
log.info("Getting document options FROM entity_changes server.");
// response is expected to contain documentId and documentSecret options
const resp = await request.exec({

2
src/services/sql_init.js
View File

@ -84,8 +84,6 @@ async function createInitialDatabase(username, password, theme) {
const zipImportService = require("./import/zip");
await zipImportService.importZip(dummyTaskContext, demoFile, rootNote);
require('./sync_table').fillAllSyncRows();
sql.transactional(() => {
const startNoteId = sql.getValue("SELECT noteId FROM branches WHERE parentNoteId = 'root' AND isDeleted = 0 ORDER BY notePosition");

56
src/services/sync.js
View File

@ -15,7 +15,7 @@ const syncMutexService = require('./sync_mutex');
const cls = require('./cls');
const request = require('./request');
const ws = require('./ws');
const syncTableService = require('./sync_table');
const entityChangesService = require('./entity_changes.js');
const entityConstructor = require('../entities/entity_constructor');
let proxyToggle = true;
@ -113,10 +113,10 @@ async function doLogin() {
// this is important in a scenario where we setup the sync by manually copying the document
// lastSyncedPull then could be pretty off for the newly cloned client
if (lastSyncedPull > resp.maxSyncId) {
log.info(`Lowering last synced pull from ${lastSyncedPull} to ${resp.maxSyncId}`);
if (lastSyncedPull > resp.maxEntityChangeId) {
log.info(`Lowering last synced pull from ${lastSyncedPull} to ${resp.maxEntityChangeId}`);
setLastSyncedPull(resp.maxSyncId);
setLastSyncedPull(resp.maxEntityChangeId);
}
return syncContext;
@ -127,7 +127,7 @@ async function pullSync(syncContext) {
while (true) {
const lastSyncedPull = getLastSyncedPull();
const changesUri = '/api/sync/changed?lastSyncId=' + lastSyncedPull;
const changesUri = '/api/sync/changed?lastEntityChangeId=' + lastSyncedPull;
const startDate = Date.now();
@ -135,7 +135,7 @@ async function pullSync(syncContext) {
const pulledDate = Date.now();
stats.outstandingPulls = resp.maxSyncId - lastSyncedPull;
stats.outstandingPulls = resp.maxEntityChangeId - lastSyncedPull;
if (stats.outstandingPulls < 0) {
stats.outstandingPulls = 0;
@ -159,13 +159,13 @@ async function pullSync(syncContext) {
syncUpdateService.updateEntity(sync, entity, syncContext.sourceId);
}
stats.outstandingPulls = resp.maxSyncId - sync.id;
stats.outstandingPulls = resp.maxEntityChangeId - sync.id;
}
setLastSyncedPull(rows[rows.length - 1].sync.id);
});
log.info(`Pulled ${rows.length} changes starting at syncId=${lastSyncedPull} in ${pulledDate - startDate}ms and applied them in ${Date.now() - pulledDate}ms, ${stats.outstandingPulls} outstanding pulls`);
log.info(`Pulled ${rows.length} changes starting at entityChangeId=${lastSyncedPull} in ${pulledDate - startDate}ms and applied them in ${Date.now() - pulledDate}ms, ${stats.outstandingPulls} outstanding pulls`);
}
if (atLeastOnePullApplied) {
@ -179,7 +179,7 @@ async function pushSync(syncContext) {
let lastSyncedPush = getLastSyncedPush();
while (true) {
const syncs = sql.getRows('SELECT * FROM sync WHERE isSynced = 1 AND id > ? LIMIT 1000', [lastSyncedPush]);
const syncs = sql.getRows('SELECT * FROM entity_changes WHERE isSynced = 1 AND id > ? LIMIT 1000', [lastSyncedPush]);
if (syncs.length === 0) {
log.info("Nothing to push");
@ -209,7 +209,7 @@ async function pushSync(syncContext) {
continue;
}
const syncRecords = getSyncRecords(filteredSyncs);
const syncRecords = getEntityChangesRecords(filteredSyncs);
const startDate = new Date();
await syncRequest(syncContext, 'PUT', '/api/sync/update', {
@ -233,13 +233,13 @@ async function checkContentHash(syncContext) {
const resp = await syncRequest(syncContext, 'GET', '/api/sync/check');
const lastSyncedPullId = getLastSyncedPull();
if (lastSyncedPullId < resp.maxSyncId) {
log.info(`There are some outstanding pulls (${lastSyncedPullId} vs. ${resp.maxSyncId}), skipping content check.`);
if (lastSyncedPullId < resp.maxEntityChangeId) {
log.info(`There are some outstanding pulls (${lastSyncedPullId} vs. ${resp.maxEntityChangeId}), skipping content check.`);
return true;
}
const notPushedSyncs = sql.getValue("SELECT EXISTS(SELECT 1 FROM sync WHERE isSynced = 1 AND id > ?)", [getLastSyncedPush()]);
const notPushedSyncs = sql.getValue("SELECT EXISTS(SELECT 1 FROM entity_changes WHERE isSynced = 1 AND id > ?)", [getLastSyncedPush()]);
if (notPushedSyncs) {
log.info(`There's ${notPushedSyncs} outstanding pushes, skipping content check.`);
@ -252,7 +252,7 @@ async function checkContentHash(syncContext) {
for (const {entityName, sector} of failedChecks) {
const entityPrimaryKey = entityConstructor.getEntityFromEntityName(entityName).primaryKeyName;
syncTableService.addEntitySyncsForSector(entityName, entityPrimaryKey, sector);
entityChangesService.addEntityChangesForSector(entityName, entityPrimaryKey, sector);
await syncRequest(syncContext, 'POST', `/api/sync/queue-sector/${entityName}/${sector}`);
}
@ -287,7 +287,7 @@ const primaryKeys = {
"attributes": "attributeId"
};
function getEntityRow(entityName, entityId) {
function getEntityChangeRow(entityName, entityId) {
if (entityName === 'note_reordering') {
return sql.getMap("SELECT branchId, notePosition FROM branches WHERE parentNoteId = ? AND isDeleted = 0", [entityId]);
}
@ -316,20 +316,20 @@ function getEntityRow(entityName, entityId) {
}
}
function getSyncRecords(syncs) {
function getEntityChangesRecords(entityChanges) {
const records = [];
let length = 0;
for (const sync of syncs) {
const entity = getEntityRow(sync.entityName, sync.entityId);
for (const entityChange of entityChanges) {
const entity = getEntityChangeRow(entityChange.entityName, entityChange.entityId);
if (sync.entityName === 'options' && !entity.isSynced) {
records.push({sync});
if (entityChange.entityName === 'options' && !entity.isSynced) {
records.push({entityChange});
continue;
}
const record = { sync, entity };
const record = { entityChange, entity };
records.push(record);
@ -347,8 +347,8 @@ function getLastSyncedPull() {
return parseInt(optionService.getOption('lastSyncedPull'));
}
function setLastSyncedPull(syncId) {
optionService.setOption('lastSyncedPull', syncId);
function setLastSyncedPull(entityChangeId) {
optionService.setOption('lastSyncedPull', entityChangeId);
}
function getLastSyncedPush() {
@ -363,12 +363,12 @@ function updatePushStats() {
if (syncOptions.isSyncSetup()) {
const lastSyncedPush = optionService.getOption('lastSyncedPush');
stats.outstandingPushes = sql.getValue("SELECT COUNT(1) FROM sync WHERE isSynced = 1 AND id > ?", [lastSyncedPush]);
stats.outstandingPushes = sql.getValue("SELECT COUNT(1) FROM entity_changes WHERE isSynced = 1 AND id > ?", [lastSyncedPush]);
}
}
function getMaxSyncId() {
return sql.getValue('SELECT COALESCE(MAX(id), 0) FROM sync');
function getMaxEntityChangeId() {
return sql.getValue('SELECT COALESCE(MAX(id), 0) FROM entity_changes');
}
sqlInit.dbReady.then(() => {
@ -383,7 +383,7 @@ sqlInit.dbReady.then(() => {
module.exports = {
sync,
login,
getSyncRecords,
getEntityChangesRecords,
stats,
getMaxSyncId
getMaxEntityChangeId
};

22
src/services/sync_update.js
View File

@ -1,6 +1,6 @@
const sql = require('./sql');
const log = require('./log');
const syncTableService = require('./sync_table');
const entityChangesService = require('./entity_changes.js');
const eventService = require('./events');
function updateEntity(sync, entity, sourceId) {
@ -86,7 +86,7 @@ function updateNote(remoteEntity, sourceId) {
sql.transactional(() => {
sql.replace("notes", remoteEntity);
syncTableService.addNoteSync(remoteEntity.noteId, sourceId);
entityChangesService.addNoteSync(remoteEntity.noteId, sourceId);
});
return true;
@ -104,7 +104,7 @@ function updateNoteContent(remoteEntity, sourceId) {
sql.transactional(() => {
sql.replace("note_contents", remoteEntity);
syncTableService.addNoteContentSync(remoteEntity.noteId, sourceId);
entityChangesService.addNoteContentSync(remoteEntity.noteId, sourceId);
});
return true;
@ -126,7 +126,7 @@ function updateBranch(remoteEntity, sourceId) {
sql.replace('branches', remoteEntity);
syncTableService.addBranchSync(remoteEntity.branchId, sourceId);
entityChangesService.addBranchSync(remoteEntity.branchId, sourceId);
});
return true;
@ -142,7 +142,7 @@ function updateNoteRevision(remoteEntity, sourceId) {
if (shouldWeUpdateEntity(localEntity, remoteEntity)) {
sql.replace('note_revisions', remoteEntity);
syncTableService.addNoteRevisionSync(remoteEntity.noteRevisionId, sourceId);
entityChangesService.addNoteRevisionSync(remoteEntity.noteRevisionId, sourceId);
log.info("Update/sync note revision " + remoteEntity.noteRevisionId);
}
@ -158,7 +158,7 @@ function updateNoteRevisionContent(remoteEntity, sourceId) {
sql.replace('note_revision_contents', remoteEntity);
syncTableService.addNoteRevisionContentSync(remoteEntity.noteRevisionId, sourceId);
entityChangesService.addNoteRevisionContentSync(remoteEntity.noteRevisionId, sourceId);
});
return true;
@ -173,7 +173,7 @@ function updateNoteReordering(entityId, remote, sourceId) {
sql.execute("UPDATE branches SET notePosition = ? WHERE branchId = ?", [remote[key], key]);
}
syncTableService.addNoteReorderingSync(entityId, sourceId);
entityChangesService.addNoteReorderingSync(entityId, sourceId);
});
return true;
@ -190,7 +190,7 @@ function updateOptions(remoteEntity, sourceId) {
sql.transactional(() => {
sql.replace('options', remoteEntity);
syncTableService.addOptionsSync(remoteEntity.name, sourceId, true);
entityChangesService.addOptionsSync(remoteEntity.name, sourceId, true);
});
return true;
@ -206,7 +206,7 @@ function updateRecentNotes(remoteEntity, sourceId) {
sql.transactional(() => {
sql.replace('recent_notes', remoteEntity);
syncTableService.addRecentNoteSync(remoteEntity.noteId, sourceId);
entityChangesService.addRecentNoteSync(remoteEntity.noteId, sourceId);
});
return true;
@ -222,7 +222,7 @@ function updateAttribute(remoteEntity, sourceId) {
sql.transactional(() => {
sql.replace("attributes", remoteEntity);
syncTableService.addAttributeSync(remoteEntity.attributeId, sourceId);
entityChangesService.addAttributeSync(remoteEntity.attributeId, sourceId);
});
return true;
@ -238,7 +238,7 @@ function updateApiToken(entity, sourceId) {
sql.transactional(() => {
sql.replace("api_tokens", entity);
syncTableService.addApiTokenSync(entity.apiTokenId, sourceId);
entityChangesService.addApiTokenSync(entity.apiTokenId, sourceId);
});
return true;

4
src/services/tree.js
View File

@ -3,7 +3,7 @@
const sql = require('./sql');
const repository = require('./repository');
const Branch = require('../entities/branch');
const syncTableService = require('./sync_table');
const entityChangesService = require('./entity_changes.js');
const protectedSessionService = require('./protected_session');
function getNotes(noteIds) {
@ -138,7 +138,7 @@ function sortNotesAlphabetically(parentNoteId, directoriesFirst = false) {
position += 10;
}
syncTableService.addNoteReorderingSync(parentNoteId);
entityChangesService.addNoteReorderingSync(parentNoteId);
});
}

2
src/views/desktop.ejs
View File

@ -47,7 +47,7 @@
window.glob = {
activeDialog: null,
sourceId: '<%= sourceId %>',
maxSyncIdAtLoad: <%= maxSyncIdAtLoad %>,
maxEntityChangeIdAtLoad: <%= maxEntityChangeIdAtLoad %>,
instanceName: '<%= instanceName %>',
csrfToken: '<%= csrfToken %>',
isDev: <%= isDev %>,

2
src/views/mobile.ejs
View File

@ -111,7 +111,7 @@
window.glob = {
activeDialog: null,
sourceId: '<%= sourceId %>',
maxSyncIdAtLoad: <%= maxSyncIdAtLoad %>,
maxEntityChangeIdAtLoad: <%= maxEntityChangeIdAtLoad %>,
instanceName: '<%= instanceName %>',
csrfToken: '<%= csrfToken %>',
isDev: <%= isDev %>,