diff --git a/apps/server/src/assets/db/migrations/0215__content_structure.sql b/apps/server/src/assets/db/migrations/0215__content_structure.sql deleted file mode 100644 index da4afcf6b..000000000 --- a/apps/server/src/assets/db/migrations/0215__content_structure.sql +++ /dev/null @@ -1,13 +0,0 @@ -CREATE TABLE IF NOT EXISTS "blobs" ( - `blobId` TEXT NOT NULL, - `content` TEXT NULL DEFAULT NULL, - `dateModified` TEXT NOT NULL, - `utcDateModified` TEXT NOT NULL, - PRIMARY KEY(`blobId`) -); - -ALTER TABLE notes ADD blobId TEXT DEFAULT NULL; -ALTER TABLE note_revisions ADD blobId TEXT DEFAULT NULL; - -CREATE INDEX IF NOT EXISTS IDX_notes_blobId on notes (blobId); -CREATE INDEX IF NOT EXISTS IDX_note_revisions_blobId on note_revisions (blobId); diff --git a/apps/server/src/assets/db/migrations/0217__drop_content_tables.sql b/apps/server/src/assets/db/migrations/0217__drop_content_tables.sql deleted file mode 100644 index 549fd6919..000000000 --- a/apps/server/src/assets/db/migrations/0217__drop_content_tables.sql +++ /dev/null @@ -1,4 +0,0 @@ -DROP TABLE note_contents; -DROP TABLE note_revision_contents; - -DELETE FROM entity_changes WHERE entityName IN ('note_contents', 'note_revision_contents'); diff --git a/apps/server/src/assets/db/migrations/0218__rename_note_revision_to_revision.sql b/apps/server/src/assets/db/migrations/0218__rename_note_revision_to_revision.sql deleted file mode 100644 index c67c52393..000000000 --- a/apps/server/src/assets/db/migrations/0218__rename_note_revision_to_revision.sql +++ /dev/null @@ -1,26 +0,0 @@ -CREATE TABLE IF NOT EXISTS "revisions" (`revisionId` TEXT NOT NULL PRIMARY KEY, - `noteId` TEXT NOT NULL, - type TEXT DEFAULT '' NOT NULL, - mime TEXT DEFAULT '' NOT NULL, - `title` TEXT NOT NULL, - `isProtected` INT NOT NULL DEFAULT 0, - blobId TEXT DEFAULT NULL, - `utcDateLastEdited` TEXT NOT NULL, - `utcDateCreated` TEXT NOT NULL, - `utcDateModified` TEXT NOT NULL, - `dateLastEdited` TEXT NOT NULL, - `dateCreated` TEXT NOT NULL); - -INSERT INTO revisions (revisionId, noteId, type, mime, title, isProtected, utcDateLastEdited, utcDateCreated, utcDateModified, dateLastEdited, dateCreated, blobId) -SELECT noteRevisionId, noteId, type, mime, title, isProtected, utcDateLastEdited, utcDateCreated, utcDateModified, dateLastEdited, dateCreated, blobId FROM note_revisions; - -DROP TABLE note_revisions; - -CREATE INDEX `IDX_revisions_noteId` ON `revisions` (`noteId`); -CREATE INDEX `IDX_revisions_utcDateCreated` ON `revisions` (`utcDateCreated`); -CREATE INDEX `IDX_revisions_utcDateLastEdited` ON `revisions` (`utcDateLastEdited`); -CREATE INDEX `IDX_revisions_dateCreated` ON `revisions` (`dateCreated`); -CREATE INDEX `IDX_revisions_dateLastEdited` ON `revisions` (`dateLastEdited`); -CREATE INDEX IF NOT EXISTS IDX_revisions_blobId on revisions (blobId); - -UPDATE entity_changes SET entityName = 'revisions' WHERE entityName = 'note_revisions'; diff --git a/apps/server/src/assets/db/migrations/0219__attachments.sql b/apps/server/src/assets/db/migrations/0219__attachments.sql deleted file mode 100644 index 246360bfe..000000000 --- a/apps/server/src/assets/db/migrations/0219__attachments.sql +++ /dev/null @@ -1,23 +0,0 @@ -CREATE TABLE IF NOT EXISTS "attachments" -( - attachmentId TEXT not null primary key, - ownerId TEXT not null, - role TEXT not null, - mime TEXT not null, - title TEXT not null, - isProtected INT not null DEFAULT 0, - position INT default 0 not null, - blobId TEXT DEFAULT null, - dateModified TEXT NOT NULL, - utcDateModified TEXT not null, - utcDateScheduledForErasureSince TEXT DEFAULT NULL, - isDeleted INT not null, - deleteId TEXT DEFAULT NULL); - -CREATE INDEX IDX_attachments_ownerId_role - on attachments (ownerId, role); - -CREATE INDEX IDX_attachments_utcDateScheduledForErasureSince - on attachments (utcDateScheduledForErasureSince); - -CREATE INDEX IF NOT EXISTS IDX_attachments_blobId on attachments (blobId); diff --git a/apps/server/src/assets/db/migrations/0221__remove_hideIncludedImages_main_option.sql b/apps/server/src/assets/db/migrations/0221__remove_hideIncludedImages_main_option.sql deleted file mode 100644 index 5e4ec083b..000000000 --- a/apps/server/src/assets/db/migrations/0221__remove_hideIncludedImages_main_option.sql +++ /dev/null @@ -1,2 +0,0 @@ -DELETE FROM options WHERE name = 'hideIncludedImages_main'; -DELETE FROM entity_changes WHERE entityName = 'options' AND entityId = 'hideIncludedImages_main'; \ No newline at end of file diff --git a/apps/server/src/assets/db/migrations/0222__rename_openTabs_to_openNoteContexts.sql b/apps/server/src/assets/db/migrations/0222__rename_openTabs_to_openNoteContexts.sql deleted file mode 100644 index c4d719bce..000000000 --- a/apps/server/src/assets/db/migrations/0222__rename_openTabs_to_openNoteContexts.sql +++ /dev/null @@ -1,2 +0,0 @@ -UPDATE options SET name = 'openNoteContexts' WHERE name = 'openTabs'; -UPDATE entity_changes SET entityId = 'openNoteContexts' WHERE entityName = 'options' AND entityId = 'openTabs'; diff --git a/apps/server/src/assets/db/migrations/0223__NOOP.sql b/apps/server/src/assets/db/migrations/0223__NOOP.sql deleted file mode 100644 index e0ac49d1e..000000000 --- a/apps/server/src/assets/db/migrations/0223__NOOP.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT 1; diff --git a/apps/server/src/assets/db/migrations/0224__fix_blobIds.sql b/apps/server/src/assets/db/migrations/0224__fix_blobIds.sql deleted file mode 100644 index 725e9d1f2..000000000 --- a/apps/server/src/assets/db/migrations/0224__fix_blobIds.sql +++ /dev/null @@ -1,14 +0,0 @@ -UPDATE blobs SET blobId = REPLACE(blobId, '+', 'X'); -UPDATE blobs SET blobId = REPLACE(blobId, '/', 'Y'); - -UPDATE notes SET blobId = REPLACE(blobId, '+', 'X'); -UPDATE notes SET blobId = REPLACE(blobId, '/', 'Y'); - -UPDATE attachments SET blobId = REPLACE(blobId, '+', 'X'); -UPDATE attachments SET blobId = REPLACE(blobId, '/', 'Y'); - -UPDATE revisions SET blobId = REPLACE(blobId, '+', 'X'); -UPDATE revisions SET blobId = REPLACE(blobId, '/', 'Y'); - -UPDATE entity_changes SET entityId = REPLACE(entityId, '+', 'X') WHERE entityName = 'blobs'; -UPDATE entity_changes SET entityId = REPLACE(entityId, '/', 'Y') WHERE entityName = 'blobs'; diff --git a/apps/server/src/assets/db/migrations/0225__create_blobId_indices.sql b/apps/server/src/assets/db/migrations/0225__create_blobId_indices.sql deleted file mode 100644 index bd3445447..000000000 --- a/apps/server/src/assets/db/migrations/0225__create_blobId_indices.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE INDEX IF NOT EXISTS IDX_notes_blobId on notes (blobId); -CREATE INDEX IF NOT EXISTS IDX_revisions_blobId on revisions (blobId); -CREATE INDEX IF NOT EXISTS IDX_attachments_blobId on attachments (blobId); diff --git a/apps/server/src/assets/db/migrations/0226__rename_noteSize_label.sql b/apps/server/src/assets/db/migrations/0226__rename_noteSize_label.sql deleted file mode 100644 index cd2239af4..000000000 --- a/apps/server/src/assets/db/migrations/0226__rename_noteSize_label.sql +++ /dev/null @@ -1 +0,0 @@ -UPDATE attributes SET value = 'contentAndAttachmentsAndRevisionsSize' WHERE name = 'orderBy' AND value = 'noteSize'; diff --git a/apps/server/src/assets/db/migrations/0227__disable_image_compression.sql b/apps/server/src/assets/db/migrations/0227__disable_image_compression.sql deleted file mode 100644 index a5350deff..000000000 --- a/apps/server/src/assets/db/migrations/0227__disable_image_compression.sql +++ /dev/null @@ -1,2 +0,0 @@ --- emergency disabling of image compression since it appears to make problems in migration to 0.61 -UPDATE options SET value = 'false' WHERE name = 'compressImages'; diff --git a/apps/server/src/assets/db/migrations/0228__fix_blobIds.sql b/apps/server/src/assets/db/migrations/0228__fix_blobIds.sql deleted file mode 100644 index 339a6100a..000000000 --- a/apps/server/src/assets/db/migrations/0228__fix_blobIds.sql +++ /dev/null @@ -1,17 +0,0 @@ --- + is normally replaced by X and / by Y, but this can temporarily cause UNIQUE key exception --- this might create blob duplicates, but cleanup will eventually take care of it - -UPDATE blobs SET blobId = REPLACE(blobId, '+', 'A'); -UPDATE blobs SET blobId = REPLACE(blobId, '/', 'B'); - -UPDATE notes SET blobId = REPLACE(blobId, '+', 'A'); -UPDATE notes SET blobId = REPLACE(blobId, '/', 'B'); - -UPDATE attachments SET blobId = REPLACE(blobId, '+', 'A'); -UPDATE attachments SET blobId = REPLACE(blobId, '/', 'B'); - -UPDATE revisions SET blobId = REPLACE(blobId, '+', 'A'); -UPDATE revisions SET blobId = REPLACE(blobId, '/', 'B'); - -UPDATE entity_changes SET entityId = REPLACE(entityId, '+', 'A') WHERE entityName = 'blobs'; -UPDATE entity_changes SET entityId = REPLACE(entityId, '/', 'B') WHERE entityName = 'blobs'; diff --git a/apps/server/src/assets/db/migrations/0229__add_oauth_user_data_table.sql b/apps/server/src/assets/db/migrations/0229__add_oauth_user_data_table.sql deleted file mode 100644 index ea2db4ef9..000000000 --- a/apps/server/src/assets/db/migrations/0229__add_oauth_user_data_table.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Add the oauth user data table -CREATE TABLE IF NOT EXISTS "user_data" -( - tmpID INT, - username TEXT, - email TEXT, - userIDEncryptedDataKey TEXT, - userIDVerificationHash TEXT, - salt TEXT, - derivedKey TEXT, - isSetup TEXT DEFAULT "false", - UNIQUE (tmpID), - PRIMARY KEY (tmpID) -); \ No newline at end of file diff --git a/apps/server/src/assets/db/migrations/0230__vector_embeddings.sql b/apps/server/src/assets/db/migrations/0230__vector_embeddings.sql deleted file mode 100644 index 45f14fddf..000000000 --- a/apps/server/src/assets/db/migrations/0230__vector_embeddings.sql +++ /dev/null @@ -1,46 +0,0 @@ --- Add tables for vector embeddings storage and management --- This migration adds embedding support to the main document.db database - --- Store embeddings for notes -CREATE TABLE IF NOT EXISTS "note_embeddings" ( - "embedId" TEXT NOT NULL PRIMARY KEY, - "noteId" TEXT NOT NULL, - "providerId" TEXT NOT NULL, - "modelId" TEXT NOT NULL, - "dimension" INTEGER NOT NULL, - "embedding" BLOB NOT NULL, - "version" INTEGER NOT NULL DEFAULT 1, - "dateCreated" TEXT NOT NULL, - "utcDateCreated" TEXT NOT NULL, - "dateModified" TEXT NOT NULL, - "utcDateModified" TEXT NOT NULL -); - -CREATE INDEX "IDX_note_embeddings_noteId" ON "note_embeddings" ("noteId"); -CREATE INDEX "IDX_note_embeddings_providerId_modelId" ON "note_embeddings" ("providerId", "modelId"); - --- Table to track which notes need embedding updates -CREATE TABLE IF NOT EXISTS "embedding_queue" ( - "noteId" TEXT NOT NULL PRIMARY KEY, - "operation" TEXT NOT NULL, -- CREATE, UPDATE, DELETE - "dateQueued" TEXT NOT NULL, - "utcDateQueued" TEXT NOT NULL, - "priority" INTEGER NOT NULL DEFAULT 0, - "attempts" INTEGER NOT NULL DEFAULT 0, - "lastAttempt" TEXT NULL, - "error" TEXT NULL, - "failed" INTEGER NOT NULL DEFAULT 0, - "isProcessing" INTEGER NOT NULL DEFAULT 0 -); - --- Table to store embedding provider configurations -CREATE TABLE IF NOT EXISTS "embedding_providers" ( - "providerId" TEXT NOT NULL PRIMARY KEY, - "name" TEXT NOT NULL, - "priority" INTEGER NOT NULL DEFAULT 0, - "config" TEXT NOT NULL, -- JSON config object - "dateCreated" TEXT NOT NULL, - "utcDateCreated" TEXT NOT NULL, - "dateModified" TEXT NOT NULL, - "utcDateModified" TEXT NOT NULL -); \ No newline at end of file diff --git a/apps/server/src/assets/db/migrations/0231__session_store.sql b/apps/server/src/assets/db/migrations/0231__session_store.sql deleted file mode 100644 index de245d25c..000000000 --- a/apps/server/src/assets/db/migrations/0231__session_store.sql +++ /dev/null @@ -1,5 +0,0 @@ -CREATE TABLE IF NOT EXISTS sessions ( - id TEXT PRIMARY KEY, - data TEXT, - expires INTEGER -); \ No newline at end of file diff --git a/apps/server/src/assets/db/migrations/0216__move_content_into_blobs.ts b/apps/server/src/migrations/0216__move_content_into_blobs.ts similarity index 97% rename from apps/server/src/assets/db/migrations/0216__move_content_into_blobs.ts rename to apps/server/src/migrations/0216__move_content_into_blobs.ts index 6251e70aa..47d4c5761 100644 --- a/apps/server/src/assets/db/migrations/0216__move_content_into_blobs.ts +++ b/apps/server/src/migrations/0216__move_content_into_blobs.ts @@ -1,5 +1,5 @@ -import sql from "../../../services/sql.js"; -import utils from "../../../services/utils.js"; +import sql from "../services/sql.js"; +import utils from "../services/utils.js"; interface NoteContentsRow { noteId: string; diff --git a/apps/server/src/assets/db/migrations/0220__migrate_images_to_attachments.ts b/apps/server/src/migrations/0220__migrate_images_to_attachments.ts similarity index 76% rename from apps/server/src/assets/db/migrations/0220__migrate_images_to_attachments.ts rename to apps/server/src/migrations/0220__migrate_images_to_attachments.ts index 197246273..9e06644c3 100644 --- a/apps/server/src/assets/db/migrations/0220__migrate_images_to_attachments.ts +++ b/apps/server/src/migrations/0220__migrate_images_to_attachments.ts @@ -1,8 +1,8 @@ -import becca from "../../../becca/becca.js"; -import becca_loader from "../../../becca/becca_loader.js"; -import cls from "../../../services/cls.js"; -import log from "../../../services/log.js"; -import sql from "../../../services/sql.js"; +import becca from "../becca/becca.js"; +import becca_loader from "../becca/becca_loader.js"; +import cls from "../services/cls.js"; +import log from "../services/log.js"; +import sql from "../services/sql.js"; export default () => { cls.init(() => { diff --git a/apps/server/src/migrations/migrations.ts b/apps/server/src/migrations/migrations.ts new file mode 100644 index 000000000..1ac414afe --- /dev/null +++ b/apps/server/src/migrations/migrations.ts @@ -0,0 +1,295 @@ +/** + * @module + * + * Contains all the migrations that are run on the database. + */ + +// Migrations should be kept in descending order, so the latest migration is first. +const MIGRATIONS: (SqlMigration | JsMigration)[] = [ + // Session store + { + version: 231, + sql: /*sql*/`\ + CREATE TABLE IF NOT EXISTS sessions ( + id TEXT PRIMARY KEY, + data TEXT, + expires INTEGER + ); + ` + }, + // Add tables for vector embeddings storage and management + // This migration adds embedding support to the main document.db database + { + version: 230, + sql: /*sql*/`\ + -- Store embeddings for notes + CREATE TABLE IF NOT EXISTS "note_embeddings" ( + "embedId" TEXT NOT NULL PRIMARY KEY, + "noteId" TEXT NOT NULL, + "providerId" TEXT NOT NULL, + "modelId" TEXT NOT NULL, + "dimension" INTEGER NOT NULL, + "embedding" BLOB NOT NULL, + "version" INTEGER NOT NULL DEFAULT 1, + "dateCreated" TEXT NOT NULL, + "utcDateCreated" TEXT NOT NULL, + "dateModified" TEXT NOT NULL, + "utcDateModified" TEXT NOT NULL + ); + + CREATE INDEX "IDX_note_embeddings_noteId" ON "note_embeddings" ("noteId"); + CREATE INDEX "IDX_note_embeddings_providerId_modelId" ON "note_embeddings" ("providerId", "modelId"); + + -- Table to track which notes need embedding updates + CREATE TABLE IF NOT EXISTS "embedding_queue" ( + "noteId" TEXT NOT NULL PRIMARY KEY, + "operation" TEXT NOT NULL, -- CREATE, UPDATE, DELETE + "dateQueued" TEXT NOT NULL, + "utcDateQueued" TEXT NOT NULL, + "priority" INTEGER NOT NULL DEFAULT 0, + "attempts" INTEGER NOT NULL DEFAULT 0, + "lastAttempt" TEXT NULL, + "error" TEXT NULL, + "failed" INTEGER NOT NULL DEFAULT 0, + "isProcessing" INTEGER NOT NULL DEFAULT 0 + ); + + -- Table to store embedding provider configurations + CREATE TABLE IF NOT EXISTS "embedding_providers" ( + "providerId" TEXT NOT NULL PRIMARY KEY, + "name" TEXT NOT NULL, + "priority" INTEGER NOT NULL DEFAULT 0, + "config" TEXT NOT NULL, -- JSON config object + "dateCreated" TEXT NOT NULL, + "utcDateCreated" TEXT NOT NULL, + "dateModified" TEXT NOT NULL, + "utcDateModified" TEXT NOT NULL + ); + ` + }, + + // add the oauth user data table + { + version: 229, + sql: /*sql*/`\ + CREATE TABLE IF NOT EXISTS "user_data" + ( + tmpID INT, + username TEXT, + email TEXT, + userIDEncryptedDataKey TEXT, + userIDVerificationHash TEXT, + salt TEXT, + derivedKey TEXT, + isSetup TEXT DEFAULT "false", + UNIQUE (tmpID), + PRIMARY KEY (tmpID) + ); + ` + }, + // fix blob IDs + { + version: 228, + sql: /*sql*/`\ + -- + is normally replaced by X and / by Y, but this can temporarily cause UNIQUE key exception + -- this might create blob duplicates, but cleanup will eventually take care of it + + UPDATE blobs SET blobId = REPLACE(blobId, '+', 'A'); + UPDATE blobs SET blobId = REPLACE(blobId, '/', 'B'); + + UPDATE notes SET blobId = REPLACE(blobId, '+', 'A'); + UPDATE notes SET blobId = REPLACE(blobId, '/', 'B'); + + UPDATE attachments SET blobId = REPLACE(blobId, '+', 'A'); + UPDATE attachments SET blobId = REPLACE(blobId, '/', 'B'); + + UPDATE revisions SET blobId = REPLACE(blobId, '+', 'A'); + UPDATE revisions SET blobId = REPLACE(blobId, '/', 'B'); + + UPDATE entity_changes SET entityId = REPLACE(entityId, '+', 'A') WHERE entityName = 'blobs'; + UPDATE entity_changes SET entityId = REPLACE(entityId, '/', 'B') WHERE entityName = 'blobs'; + ` + }, + // disable image compression + { + version: 227, + sql: /*sql*/`\ + -- emergency disabling of image compression since it appears to make problems in migration to 0.61 + UPDATE options SET value = 'false' WHERE name = 'compressImages'; + ` + }, + // rename note size label + { + version: 226, + sql: /*sql*/`\ + UPDATE attributes SET value = 'contentAndAttachmentsAndRevisionsSize' WHERE name = 'orderBy' AND value = 'noteSize'; + ` + }, + // create blob ID indices + { + version: 225, + sql: /*sql*/`\ + CREATE INDEX IF NOT EXISTS IDX_notes_blobId on notes (blobId); + CREATE INDEX IF NOT EXISTS IDX_revisions_blobId on revisions (blobId); + CREATE INDEX IF NOT EXISTS IDX_attachments_blobId on attachments (blobId); + ` + }, + // fix blob IDs + { + version: 224, + sql: /*sql*/`\ + UPDATE blobs SET blobId = REPLACE(blobId, '+', 'X'); + UPDATE blobs SET blobId = REPLACE(blobId, '/', 'Y'); + + UPDATE notes SET blobId = REPLACE(blobId, '+', 'X'); + UPDATE notes SET blobId = REPLACE(blobId, '/', 'Y'); + + UPDATE attachments SET blobId = REPLACE(blobId, '+', 'X'); + UPDATE attachments SET blobId = REPLACE(blobId, '/', 'Y'); + + UPDATE revisions SET blobId = REPLACE(blobId, '+', 'X'); + UPDATE revisions SET blobId = REPLACE(blobId, '/', 'Y'); + + UPDATE entity_changes SET entityId = REPLACE(entityId, '+', 'X') WHERE entityName = 'blobs'; + UPDATE entity_changes SET entityId = REPLACE(entityId, '/', 'Y') WHERE entityName = 'blobs'; + ` + }, + // no operation + { + version: 223, + sql: /*sql*/`\ + SELECT 1; + ` + }, + // rename open tabs to open note contexts + { + version: 222, + sql: /*sql*/`\ + UPDATE options SET name = 'openNoteContexts' WHERE name = 'openTabs'; + UPDATE entity_changes SET entityId = 'openNoteContexts' WHERE entityName = 'options' AND entityId = 'openTabs'; + ` + }, + // remove hide included images option + { + version: 221, + sql: /*sql*/`\ + DELETE FROM options WHERE name = 'hideIncludedImages_main'; + DELETE FROM entity_changes WHERE entityName = 'options' AND entityId = 'hideIncludedImages_main'; + ` + }, + // migrate images to attachments + { + version: 220, + module: () => import("./0220__migrate_images_to_attachments.js") + }, + // attachments + { + version: 219, + sql: /*sql*/`\ + CREATE TABLE IF NOT EXISTS "attachments" + ( + attachmentId TEXT not null primary key, + ownerId TEXT not null, + role TEXT not null, + mime TEXT not null, + title TEXT not null, + isProtected INT not null DEFAULT 0, + position INT default 0 not null, + blobId TEXT DEFAULT null, + dateModified TEXT NOT NULL, + utcDateModified TEXT not null, + utcDateScheduledForErasureSince TEXT DEFAULT NULL, + isDeleted INT not null, + deleteId TEXT DEFAULT NULL); + + CREATE INDEX IDX_attachments_ownerId_role + on attachments (ownerId, role); + + CREATE INDEX IDX_attachments_utcDateScheduledForErasureSince + on attachments (utcDateScheduledForErasureSince); + + CREATE INDEX IF NOT EXISTS IDX_attachments_blobId on attachments (blobId); + ` + }, + // rename note revision to revision + { + version: 218, + sql: /*sql*/`\ + CREATE TABLE IF NOT EXISTS "revisions" ( + revisionId TEXT NOT NULL PRIMARY KEY, + noteId TEXT NOT NULL, + type TEXT DEFAULT '' NOT NULL, + mime TEXT DEFAULT '' NOT NULL, + title TEXT NOT NULL, + isProtected INT NOT NULL DEFAULT 0, + blobId TEXT DEFAULT NULL, + utcDateLastEdited TEXT NOT NULL, + utcDateCreated TEXT NOT NULL, + utcDateModified TEXT NOT NULL, + dateLastEdited TEXT NOT NULL, + dateCreated TEXT NOT NULL + ); + + INSERT INTO revisions (revisionId, noteId, type, mime, title, isProtected, utcDateLastEdited, utcDateCreated, utcDateModified, dateLastEdited, dateCreated, blobId) + SELECT noteRevisionId, noteId, type, mime, title, isProtected, utcDateLastEdited, utcDateCreated, utcDateModified, dateLastEdited, dateCreated, blobId FROM note_revisions; + + DROP TABLE note_revisions; + + CREATE INDEX IDX_revisions_noteId ON revisions (noteId); + CREATE INDEX IDX_revisions_utcDateCreated ON revisions (utcDateCreated); + CREATE INDEX IDX_revisions_utcDateLastEdited ON revisions (utcDateLastEdited); + CREATE INDEX IDX_revisions_dateCreated ON revisions (dateCreated); + CREATE INDEX IDX_revisions_dateLastEdited ON revisions (dateLastEdited); + CREATE INDEX IF NOT EXISTS IDX_revisions_blobId on revisions (blobId); + + UPDATE entity_changes SET entityName = 'revisions' WHERE entityName = 'note_revisions'; + ` + }, + // drop content tables + { + version: 217, + sql: /*sql*/`\ + DROP TABLE note_contents; + DROP TABLE note_revision_contents; + + DELETE FROM entity_changes WHERE entityName IN ('note_contents', 'note_revision_contents'); + ` + }, + { + version: 216, + module: async () => import("./0216__move_content_into_blobs.js") + }, + // content structure + { + version: 215, + sql: /*sql*/`\ + CREATE TABLE IF NOT EXISTS "blobs" ( + blobId TEXT NOT NULL, + content TEXT NULL DEFAULT NULL, + dateModified TEXT NOT NULL, + utcDateModified TEXT NOT NULL, + PRIMARY KEY (blobId) + ); + + ALTER TABLE notes ADD blobId TEXT DEFAULT NULL; + ALTER TABLE note_revisions ADD blobId TEXT DEFAULT NULL; + + CREATE INDEX IF NOT EXISTS IDX_notes_blobId on notes (blobId); + CREATE INDEX IF NOT EXISTS IDX_note_revisions_blobId on note_revisions (blobId); + ` + } +]; + +export default MIGRATIONS; + +interface Migration { + version: number; +} + +interface SqlMigration extends Migration { + sql: string; +} + +interface JsMigration extends Migration { + module: () => Promise<{ default: () => void }>; +} diff --git a/apps/server/src/services/migration.ts b/apps/server/src/services/migration.ts index 2271b5de4..57ad890c6 100644 --- a/apps/server/src/services/migration.ts +++ b/apps/server/src/services/migration.ts @@ -1,25 +1,19 @@ import backupService from "./backup.js"; import sql from "./sql.js"; -import fs from "fs"; import log from "./log.js"; import { crash } from "./utils.js"; -import resourceDir from "./resource_dir.js"; import appInfo from "./app_info.js"; import cls from "./cls.js"; import { t } from "i18next"; -import { join } from "path"; +import MIGRATIONS from "../migrations/migrations.js"; interface MigrationInfo { dbVersion: number; - name: string; - file: string; - type: "sql" | "js" | "ts" | string; /** - * Contains the JavaScript/TypeScript migration as a callback method that must be called to trigger the migration. - * The method cannot be async since it runs in an SQL transaction. - * For SQL migrations, this value is falsy. + * If string, then the migration is an SQL script that will be executed. + * If a function, then the migration is a JavaScript/TypeScript module that will be executed. */ - module?: () => void; + migration: string | (() => void); } async function migrate() { @@ -37,7 +31,6 @@ async function migrate() { ); const migrations = await prepareMigrations(currentDbVersion); - migrations.sort((a, b) => a.dbVersion - b.dbVersion); // all migrations are executed in one transaction - upgrade either succeeds, or the user can stay at the old version // otherwise if half of the migrations succeed, user can't use any version - DB is too "new" for the old app, @@ -76,53 +69,37 @@ async function migrate() { } async function prepareMigrations(currentDbVersion: number): Promise { - const migrationFiles = fs.readdirSync(resourceDir.MIGRATIONS_DIR) ?? []; + MIGRATIONS.sort((a, b) => a.version - b.version); const migrations: MigrationInfo[] = []; - for (const file of migrationFiles) { - const match = file.match(/^([0-9]{4})__([a-zA-Z0-9_ ]+)\.(sql|js|ts)$/); - if (!match) { - continue; - } - - const dbVersion = parseInt(match[1]); + for (const migration of MIGRATIONS) { + const dbVersion = migration.version; if (dbVersion > currentDbVersion) { - const name = match[2]; - const type = match[3]; - - const migration: MigrationInfo = { - dbVersion: dbVersion, - name: name, - file: file, - type: type - }; - - if (type === "js" || type === "ts") { + if ("sql" in migration) { + migrations.push({ + dbVersion, + migration: migration.sql + }); + } else { // Due to ESM imports, the migration file needs to be imported asynchronously and thus cannot be loaded at migration time (since migration is not asynchronous). // As such we have to preload the ESM. - // Going back to the original approach but making it webpack-compatible - const importPath = join(resourceDir.MIGRATIONS_DIR, file); - migration.module = (await import(importPath)).default; + migrations.push({ + dbVersion, + migration: (await migration.module()).default + }); } - - migrations.push(migration); } } return migrations; } -function executeMigration(mig: MigrationInfo) { - if (mig.module) { - console.log("Migration with JS module"); - mig.module(); - } else if (mig.type === "sql") { - const migrationSql = fs.readFileSync(`${resourceDir.MIGRATIONS_DIR}/${mig.file}`).toString("utf8"); - - console.log(`Migration with SQL script: ${migrationSql}`); - - sql.executeScript(migrationSql); +function executeMigration({ migration }: MigrationInfo) { + if (typeof migration === "string") { + console.log(`Migration with SQL script: ${migration}`); + sql.executeScript(migration); } else { - throw new Error(`Unknown migration type '${mig.type}'`); - } + console.log("Migration with JS module"); + migration(); + }; } function getDbVersion() { diff --git a/apps/server/src/services/resource_dir.ts b/apps/server/src/services/resource_dir.ts index a336bd3b8..c274b43ce 100644 --- a/apps/server/src/services/resource_dir.ts +++ b/apps/server/src/services/resource_dir.ts @@ -14,16 +14,8 @@ if (!fs.existsSync(DB_INIT_DIR)) { process.exit(1); } -const MIGRATIONS_DIR = path.resolve(DB_INIT_DIR, "migrations"); - -if (!fs.existsSync(MIGRATIONS_DIR)) { - log.error(`Could not find migration directory: ${MIGRATIONS_DIR}`); - process.exit(1); -} - export default { RESOURCE_DIR, - MIGRATIONS_DIR, DB_INIT_DIR, ELECTRON_APP_ROOT_DIR };