From c435b97e10cc05cf20174f5bd7cb34cfc986487c Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 10:54:29 +0300 Subject: [PATCH 01/29] docker: Fix "Missing script: "rebuild"" --- package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/package.json b/package.json index 9e6506d8a..e37e6204c 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,6 @@ "start-test-server": "npm run qswitch-server; rm -rf ./data-test; cross-env TRILIUM_SAFE_MODE=1 TRILIUM_DATA_DIR=./data-test TRILIUM_SYNC_SERVER_HOST=http://tsyncserver:4000 TRILIUM_ENV=dev TRILIUM_PORT=9999 ts-node src/www.ts", "switch-server": "rm -rf ./node_modules/better-sqlite3 && npm install", "switch-electron": "./node_modules/.bin/electron-rebuild", - "rebuild": "electron-rebuild", "qswitch-server": "rm -rf ./node_modules/better-sqlite3/bin ; mkdir -p ./node_modules/better-sqlite3/build ; cp ./bin/better-sqlite3/linux-server-better_sqlite3.node ./node_modules/better-sqlite3/build/better_sqlite3.node", "qswitch-electron": "rm -rf ./node_modules/better-sqlite3/bin ; mkdir -p ./node_modules/better-sqlite3/build ; cp ./bin/better-sqlite3/linux-desktop-better_sqlite3.node ./node_modules/better-sqlite3/build/better_sqlite3.node", "build-backend-docs": "rm -rf ./docs/backend_api && ./node_modules/.bin/jsdoc -c jsdoc-conf.json -d ./docs/backend_api src/becca/entities/*.js src/services/backend_script_api.js src/services/sql.js", @@ -32,7 +31,7 @@ "test-jasmine": "TRILIUM_DATA_DIR=~/trilium/data-test jasmine", "test-es6": "node -r esm spec-es6/attribute_parser.spec.js ", "test": "npm run test-jasmine && npm run test-es6", - "postinstall": "rimraf ./node_modules/canvas && npm run rebuild" + "postinstall": "rimraf ./node_modules/canvas" }, "dependencies": { "@braintree/sanitize-url": "6.0.4", From 408369fcac3206722708c17cb519fa1a29f336dc Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 10:54:37 +0300 Subject: [PATCH 02/29] docker: Add build directory to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0e095b075..2a427512e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .DS_Store node_modules/ dist/ +build/ src/public/app-dist/ npm-debug.log yarn-error.log From 0a7fcf399fb854a29c405d2d79f579d5a910bcc1 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 11:06:45 +0300 Subject: [PATCH 03/29] docker: Fix webpack --- bin/copy-trilium.sh | 2 +- package.json | 2 +- webpack.config.js => webpack.config.ts | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename webpack.config.js => webpack.config.ts (80%) diff --git a/bin/copy-trilium.sh b/bin/copy-trilium.sh index 9fdfe0293..79b823e31 100755 --- a/bin/copy-trilium.sh +++ b/bin/copy-trilium.sh @@ -24,7 +24,7 @@ done for f in 'package.json' 'package-lock.json' 'README.md' 'LICENSE' 'config-sample.ini' 'electron.js'; do cp "$f" "$DIR"/ done -cp webpack-* "$DIR"/ # here warning because there is no 'webpack-*', but webpack.config.js only +cp webpack-* "$DIR"/ # here warning because there is no 'webpack-*', but webpack.config.ts only # run in subshell (so we return to original dir) (cd $DIR && n exec 18.18.2 npm install --only=prod) diff --git a/package.json b/package.json index e37e6204c..e48bff6c2 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "build-backend-docs": "rm -rf ./docs/backend_api && ./node_modules/.bin/jsdoc -c jsdoc-conf.json -d ./docs/backend_api src/becca/entities/*.js src/services/backend_script_api.js src/services/sql.js", "build-frontend-docs": "rm -rf ./docs/frontend_api && ./node_modules/.bin/jsdoc -c jsdoc-conf.json -d ./docs/frontend_api src/public/app/entities/*.js src/public/app/services/frontend_script_api.js src/public/app/widgets/basic_widget.js src/public/app/widgets/note_context_aware_widget.js src/public/app/widgets/right_panel_widget.js", "build-docs": "npm run build-backend-docs && npm run build-frontend-docs", - "webpack": "webpack -c webpack.config.js", + "webpack": "webpack -c webpack.config.ts", "test-jasmine": "TRILIUM_DATA_DIR=~/trilium/data-test jasmine", "test-es6": "node -r esm spec-es6/attribute_parser.spec.js ", "test": "npm run test-jasmine && npm run test-es6", diff --git a/webpack.config.js b/webpack.config.ts similarity index 80% rename from webpack.config.js rename to webpack.config.ts index 2e3bd7b17..6f266033c 100644 --- a/webpack.config.js +++ b/webpack.config.ts @@ -1,10 +1,10 @@ const path = require('path'); -const assetPath = require('./src/services/asset_path'); +import assetPath = require('./src/services/asset_path'); module.exports = { mode: 'production', entry: { - setup: './src/public/app/setup.ts', + setup: './src/public/app/setup.js', mobile: './src/public/app/mobile.js', desktop: './src/public/app/desktop.js', }, From 9b88228be996d9d2cfd06647be4f857cd89d0c05 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 11:14:32 +0300 Subject: [PATCH 04/29] docker: Cache system dependencies installation --- Dockerfile | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index d8cb9b0c3..89bda6741 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,18 @@ # !!! Don't try to build this Dockerfile directly, run it through bin/build-docker.sh script !!! FROM node:18.18.2-alpine +# Configure system dependencies +RUN apk add --no-cache --virtual .build-dependencies \ + autoconf \ + automake \ + g++ \ + gcc \ + libtool \ + make \ + nasm \ + libpng-dev \ + python3 + # Create app directory WORKDIR /usr/src/app @@ -11,16 +23,6 @@ COPY server-package.json package.json # Install app dependencies RUN set -x \ - && apk add --no-cache --virtual .build-dependencies \ - autoconf \ - automake \ - g++ \ - gcc \ - libtool \ - make \ - nasm \ - libpng-dev \ - python3 \ && npm install \ && apk del .build-dependencies \ && npm run webpack \ From 60d134a89b457e3c47a93c24fa475e18fd11409c Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 11:32:02 +0300 Subject: [PATCH 05/29] Revert "docker: Fix webpack" This reverts commit 0a7fcf399fb854a29c405d2d79f579d5a910bcc1. --- bin/copy-trilium.sh | 2 +- package.json | 2 +- webpack.config.ts => webpack.config.js | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename webpack.config.ts => webpack.config.js (80%) diff --git a/bin/copy-trilium.sh b/bin/copy-trilium.sh index 79b823e31..9fdfe0293 100755 --- a/bin/copy-trilium.sh +++ b/bin/copy-trilium.sh @@ -24,7 +24,7 @@ done for f in 'package.json' 'package-lock.json' 'README.md' 'LICENSE' 'config-sample.ini' 'electron.js'; do cp "$f" "$DIR"/ done -cp webpack-* "$DIR"/ # here warning because there is no 'webpack-*', but webpack.config.ts only +cp webpack-* "$DIR"/ # here warning because there is no 'webpack-*', but webpack.config.js only # run in subshell (so we return to original dir) (cd $DIR && n exec 18.18.2 npm install --only=prod) diff --git a/package.json b/package.json index e48bff6c2..e37e6204c 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "build-backend-docs": "rm -rf ./docs/backend_api && ./node_modules/.bin/jsdoc -c jsdoc-conf.json -d ./docs/backend_api src/becca/entities/*.js src/services/backend_script_api.js src/services/sql.js", "build-frontend-docs": "rm -rf ./docs/frontend_api && ./node_modules/.bin/jsdoc -c jsdoc-conf.json -d ./docs/frontend_api src/public/app/entities/*.js src/public/app/services/frontend_script_api.js src/public/app/widgets/basic_widget.js src/public/app/widgets/note_context_aware_widget.js src/public/app/widgets/right_panel_widget.js", "build-docs": "npm run build-backend-docs && npm run build-frontend-docs", - "webpack": "webpack -c webpack.config.ts", + "webpack": "webpack -c webpack.config.js", "test-jasmine": "TRILIUM_DATA_DIR=~/trilium/data-test jasmine", "test-es6": "node -r esm spec-es6/attribute_parser.spec.js ", "test": "npm run test-jasmine && npm run test-es6", diff --git a/webpack.config.ts b/webpack.config.js similarity index 80% rename from webpack.config.ts rename to webpack.config.js index 6f266033c..2e3bd7b17 100644 --- a/webpack.config.ts +++ b/webpack.config.js @@ -1,10 +1,10 @@ const path = require('path'); -import assetPath = require('./src/services/asset_path'); +const assetPath = require('./src/services/asset_path'); module.exports = { mode: 'production', entry: { - setup: './src/public/app/setup.js', + setup: './src/public/app/setup.ts', mobile: './src/public/app/mobile.js', desktop: './src/public/app/desktop.js', }, From d18bfb9d69be4fdb9a63e3d86a32ce0584dd4ab7 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 11:40:52 +0300 Subject: [PATCH 06/29] docker: Fix build by generating artifacts --- .dockerignore | 2 ++ Dockerfile | 4 ++++ bin/build-docker.sh | 3 +++ tsconfig.json | 3 +-- webpack.config.js | 2 +- 5 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.dockerignore b/.dockerignore index 6c3840719..941905c66 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,3 +5,5 @@ /docs /npm-debug.log node_modules + +src/**/*.ts \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 89bda6741..78429488a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,10 @@ COPY . . COPY server-package.json package.json +# Copy TypeScript build artifacts into the original directory structure. +RUN ls +RUN cp -R build/src/* src/. + # Install app dependencies RUN set -x \ && npm install \ diff --git a/bin/build-docker.sh b/bin/build-docker.sh index c77850eba..2b3794677 100755 --- a/bin/build-docker.sh +++ b/bin/build-docker.sh @@ -5,6 +5,9 @@ SERIES=${VERSION:0:4}-latest cat package.json | grep -v electron > server-package.json +echo "Compiling typescript..." +npx tsc + sudo docker build -t zadam/trilium:$VERSION --network host -t zadam/trilium:$SERIES . if [[ $VERSION != *"beta"* ]]; then diff --git a/tsconfig.json b/tsconfig.json index bf97f7be2..293c66f7c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -13,8 +13,7 @@ }, "include": [ "./src/**/*.js", - "./src/**/*.ts", - "./*.ts" + "./src/**/*.ts" ], "exclude": ["./node_modules/**/*"], "ts-node": { diff --git a/webpack.config.js b/webpack.config.js index 2e3bd7b17..41077c00e 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -4,7 +4,7 @@ const assetPath = require('./src/services/asset_path'); module.exports = { mode: 'production', entry: { - setup: './src/public/app/setup.ts', + setup: './src/public/app/setup.js', mobile: './src/public/app/mobile.js', desktop: './src/public/app/desktop.js', }, From 7fd63e80649d6b1bec33db6b0f3a96535b254983 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 11:45:20 +0300 Subject: [PATCH 07/29] docker: Remove unneeded build directory --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 78429488a..dfeb4a0d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,7 @@ COPY server-package.json package.json # Copy TypeScript build artifacts into the original directory structure. RUN ls RUN cp -R build/src/* src/. +RUN rm -r build # Install app dependencies RUN set -x \ From ef92d17272a093fc29749d897e6ae64922987584 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 14:02:19 +0300 Subject: [PATCH 08/29] Revert change to tsconfig.json --- tsconfig.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tsconfig.json b/tsconfig.json index 293c66f7c..bf97f7be2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -13,7 +13,8 @@ }, "include": [ "./src/**/*.js", - "./src/**/*.ts" + "./src/**/*.ts", + "./*.ts" ], "exclude": ["./node_modules/**/*"], "ts-node": { From 1b0690ddfcaf558522e0cc7033c2b17956b59555 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 16:43:33 +0300 Subject: [PATCH 09/29] server: Fix note type import A regression caused by the port to TypeScript caused all note types to be treated as a "text" instead of other types such as canvas. The MIME type, however, was unaffected. --- src/becca/entities/rows.ts | 3 ++- src/services/import/zip.ts | 13 +++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/becca/entities/rows.ts b/src/becca/entities/rows.ts index 4428b6dde..4258c3805 100644 --- a/src/becca/entities/rows.ts +++ b/src/becca/entities/rows.ts @@ -91,7 +91,8 @@ export interface BranchRow { * end user. Those types should be used only for checking against, they are * not for direct use. */ -export type NoteType = ("file" | "image" | "search" | "noteMap" | "launcher" | "doc" | "contentWidget" | "text" | "relationMap" | "render" | "canvas" | "mermaid" | "book" | "webView" | "code"); +export const ALLOWED_NOTE_TYPES = [ "file", "image", "search", "noteMap", "launcher", "doc", "contentWidget", "text", "relationMap", "render", "canvas", "mermaid", "book", "webView", "code" ] as const; +export type NoteType = typeof ALLOWED_NOTE_TYPES[number]; export interface NoteRow { noteId: string; diff --git a/src/services/import/zip.ts b/src/services/import/zip.ts index 93ccc2da9..203b36442 100644 --- a/src/services/import/zip.ts +++ b/src/services/import/zip.ts @@ -20,7 +20,7 @@ import BNote = require('../../becca/entities/bnote'); import NoteMeta = require('../meta/note_meta'); import AttributeMeta = require('../meta/attribute_meta'); import { Stream } from 'stream'; -import { NoteType } from '../../becca/entities/rows'; +import { ALLOWED_NOTE_TYPES, NoteType } from '../../becca/entities/rows'; interface MetaFile { files: NoteMeta[] @@ -231,7 +231,7 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo if (!parentNoteId) { throw new Error("Missing parent note ID."); } - + const {note} = noteService.createNewNote({ parentNoteId: parentNoteId, title: noteTitle || "", @@ -462,13 +462,14 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo } let type = resolveNoteType(noteMeta?.type); + console.log("Resolved note type is ", noteMeta?.type, resolveNoteType(noteMeta?.type)); if (type !== 'file' && type !== 'image') { content = content.toString("utf-8"); } const noteTitle = utils.getNoteTitle(filePath, taskContext.data?.replaceUnderscoresWithSpaces || false, noteMeta); - + content = processNoteContent(noteMeta, type, mime, content, noteTitle || "", filePath); let note = becca.getNote(noteId); @@ -653,7 +654,11 @@ function resolveNoteType(type: string | undefined): NoteType { return 'webView'; } - return "text"; + if (type && (ALLOWED_NOTE_TYPES as readonly string[]).includes(type)) { + return type as NoteType; + } else { + return "text"; + } } export = { From d99cc11d8bfc9efc8e17e00dcec6b351a46beaf8 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 16:52:31 +0300 Subject: [PATCH 10/29] server: Fix import of notes with type=file --- src/becca/entities/rows.ts | 2 +- src/services/import/zip.ts | 4 ---- src/services/note-interface.ts | 2 +- src/services/search/expressions/note_content_fulltext.ts | 4 ++-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/becca/entities/rows.ts b/src/becca/entities/rows.ts index 4258c3805..119d392ec 100644 --- a/src/becca/entities/rows.ts +++ b/src/becca/entities/rows.ts @@ -107,5 +107,5 @@ export interface NoteRow { dateModified: string; utcDateCreated: string; utcDateModified: string; - content?: string; + content?: string | Buffer; } diff --git a/src/services/import/zip.ts b/src/services/import/zip.ts index 203b36442..f8391350b 100644 --- a/src/services/import/zip.ts +++ b/src/services/import/zip.ts @@ -500,10 +500,6 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo } } else { - if (typeof content !== "string") { - throw new Error("Incorrect content type."); - } - ({note} = noteService.createNewNote({ parentNoteId: parentNoteId, title: noteTitle || "", diff --git a/src/services/note-interface.ts b/src/services/note-interface.ts index 0b99c31ef..2cfa0b86a 100644 --- a/src/services/note-interface.ts +++ b/src/services/note-interface.ts @@ -7,7 +7,7 @@ export interface NoteParams { parentNoteId: string; templateNoteId?: string; title: string; - content: string; + content: string | Buffer; /** text, code, file, image, search, book, relationMap, canvas, webView */ type: NoteType; /** default value is derived from default mimes for type */ diff --git a/src/services/search/expressions/note_content_fulltext.ts b/src/services/search/expressions/note_content_fulltext.ts index ada9705a2..02626bf16 100644 --- a/src/services/search/expressions/note_content_fulltext.ts +++ b/src/services/search/expressions/note_content_fulltext.ts @@ -74,7 +74,7 @@ class NoteContentFulltextExp extends Expression { } if (isProtected) { - if (!protectedSessionService.isProtectedSessionAvailable() || !content) { + if (!protectedSessionService.isProtectedSessionAvailable() || !content || typeof content !== "string") { return; } @@ -86,7 +86,7 @@ class NoteContentFulltextExp extends Expression { } } - if (!content) { + if (!content || typeof content !== "string") { return; } From 606490a611c95bec1ee737887cad7a2344e79d0a Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 16:55:20 +0300 Subject: [PATCH 11/29] server: Remove log and fix whitespace --- src/services/import/zip.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/services/import/zip.ts b/src/services/import/zip.ts index f8391350b..0c4008ddf 100644 --- a/src/services/import/zip.ts +++ b/src/services/import/zip.ts @@ -231,7 +231,7 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo if (!parentNoteId) { throw new Error("Missing parent note ID."); } - + const {note} = noteService.createNewNote({ parentNoteId: parentNoteId, title: noteTitle || "", @@ -462,14 +462,13 @@ async function importZip(taskContext: TaskContext, fileBuffer: Buffer, importRoo } let type = resolveNoteType(noteMeta?.type); - console.log("Resolved note type is ", noteMeta?.type, resolveNoteType(noteMeta?.type)); if (type !== 'file' && type !== 'image') { content = content.toString("utf-8"); } const noteTitle = utils.getNoteTitle(filePath, taskContext.data?.replaceUnderscoresWithSpaces || false, noteMeta); - + content = processNoteContent(noteMeta, type, mime, content, noteTitle || "", filePath); let note = becca.getNote(noteId); From 45f26910538024f1799fdcd0f3f4aebfd10021b2 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 18:47:53 +0300 Subject: [PATCH 12/29] ci: Disable original CI workflows --- .github/{workflows => workflows_old}/codeql-analysis.yml | 0 .github/{workflows => workflows_old}/docker.yaml | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename .github/{workflows => workflows_old}/codeql-analysis.yml (100%) rename .github/{workflows => workflows_old}/docker.yaml (100%) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows_old/codeql-analysis.yml similarity index 100% rename from .github/workflows/codeql-analysis.yml rename to .github/workflows_old/codeql-analysis.yml diff --git a/.github/workflows/docker.yaml b/.github/workflows_old/docker.yaml similarity index 100% rename from .github/workflows/docker.yaml rename to .github/workflows_old/docker.yaml From 19550eec6aa5959197d100e9e720f1863f2b4077 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 18:50:34 +0300 Subject: [PATCH 13/29] ci: Set up Docker build in dev workflow --- .github/workflows/dev.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/workflows/dev.yml diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml new file mode 100644 index 000000000..ed62e70a9 --- /dev/null +++ b/.github/workflows/dev.yml @@ -0,0 +1,15 @@ +name: Dev +on: + push: +jobs: + build_docker: + name: Build Docker image + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v1 + - uses: docker/build-push-action@v2 + with: + context: . + cache-from: type=gha + cache-to: type=gha,mode=max \ No newline at end of file From c0cc1e3b52dca748497af25bfe6651d1373affde Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 18:56:18 +0300 Subject: [PATCH 14/29] ci: Run tsc & create server-package.json --- .github/workflows/dev.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index ed62e70a9..c26c28c26 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -7,6 +7,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Run the TypeScript build + run: npx tsc + - name: Create server-package.json + run: cat package.json | grep -v electron > server-package.json - uses: docker/setup-buildx-action@v1 - uses: docker/build-push-action@v2 with: From 8bcff491983827fb72602d580ea1596f84f87751 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 18:57:13 +0300 Subject: [PATCH 15/29] ci: Install npm dependencies --- .github/workflows/dev.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index c26c28c26..da72a02f1 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -7,6 +7,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Install dependencies + run: npm i - name: Run the TypeScript build run: npx tsc - name: Create server-package.json From 543c29e539f867caa6912e0485cb42f9c2edd0d4 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 19:02:51 +0300 Subject: [PATCH 16/29] server: Change build directory --- tsconfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsconfig.json b/tsconfig.json index bf97f7be2..f0f1a3fc2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,7 @@ "moduleResolution": "Node", "declaration": false, "sourceMap": true, - "outDir": "./dist", + "outDir": "./build", "strict": true, "noImplicitAny": true, "resolveJsonModule": true, From 439bf410ac9df563bcc70e5d623adc82d20ebaff Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 19:05:39 +0300 Subject: [PATCH 17/29] ci: Use newer versions of actions --- .github/workflows/dev.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index da72a02f1..82b92d278 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -13,8 +13,8 @@ jobs: run: npx tsc - name: Create server-package.json run: cat package.json | grep -v electron > server-package.json - - uses: docker/setup-buildx-action@v1 - - uses: docker/build-push-action@v2 + - uses: docker/setup-buildx-action@v3 + - uses: docker/build-push-action@v6 with: context: . cache-from: type=gha From be239d2230f58bb7253f59af7984944acad84db0 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 19:08:48 +0300 Subject: [PATCH 18/29] ci: Set up node environment & cache --- .github/workflows/dev.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 82b92d278..392046d2c 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -7,8 +7,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Install dependencies - run: npm i + - name: Set up node & dependencies + uses: actions/setup-node@v4 + with: + node-version: 18 + cache: "npm" + - run: npm ci - name: Run the TypeScript build run: npx tsc - name: Create server-package.json From 7184c648ab313ff4690fafce7bd03b7cc350bfd0 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 19:37:30 +0300 Subject: [PATCH 19/29] ci: Test Docker publishing --- .github/workflows/main.yml | 57 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 000000000..096850024 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,57 @@ +name: Main +on: + push: + branches: + - 'develop' + - 'feature/basic_ci' +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} +jobs: + build_docker: + name: Build Docker image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + attestations: write + id-token: write + steps: + - uses: actions/checkout@v4 + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: Set up node & dependencies + uses: actions/setup-node@v4 + with: + node-version: 18 + cache: "npm" + - run: npm ci + - name: Run the TypeScript build + run: npx tsc + - name: Create server-package.json + run: cat package.json | grep -v electron > server-package.json + - uses: docker/setup-buildx-action@v3 + - uses: docker/build-push-action@v6 + id: push + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true \ No newline at end of file From 1652a7a2ec80ec2ee8fc17ebd881a700246de0fb Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 20:04:00 +0300 Subject: [PATCH 20/29] ci: Remove test branch --- .github/workflows/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 096850024..747432396 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,7 +3,6 @@ on: push: branches: - 'develop' - - 'feature/basic_ci' env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} From 947aaa61564775b8f15e62563f45b2eafe0dfeac Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 21:10:40 +0300 Subject: [PATCH 21/29] webpack: Switch to TypeScript-based config --- package.json | 2 +- webpack.config.js => webpack.config.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename webpack.config.js => webpack.config.ts (82%) diff --git a/package.json b/package.json index e37e6204c..e48bff6c2 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "build-backend-docs": "rm -rf ./docs/backend_api && ./node_modules/.bin/jsdoc -c jsdoc-conf.json -d ./docs/backend_api src/becca/entities/*.js src/services/backend_script_api.js src/services/sql.js", "build-frontend-docs": "rm -rf ./docs/frontend_api && ./node_modules/.bin/jsdoc -c jsdoc-conf.json -d ./docs/frontend_api src/public/app/entities/*.js src/public/app/services/frontend_script_api.js src/public/app/widgets/basic_widget.js src/public/app/widgets/note_context_aware_widget.js src/public/app/widgets/right_panel_widget.js", "build-docs": "npm run build-backend-docs && npm run build-frontend-docs", - "webpack": "webpack -c webpack.config.js", + "webpack": "webpack -c webpack.config.ts", "test-jasmine": "TRILIUM_DATA_DIR=~/trilium/data-test jasmine", "test-es6": "node -r esm spec-es6/attribute_parser.spec.js ", "test": "npm run test-jasmine && npm run test-es6", diff --git a/webpack.config.js b/webpack.config.ts similarity index 82% rename from webpack.config.js rename to webpack.config.ts index 41077c00e..bd7135e24 100644 --- a/webpack.config.js +++ b/webpack.config.ts @@ -1,5 +1,5 @@ -const path = require('path'); -const assetPath = require('./src/services/asset_path'); +import path = require('path'); +import assetPath = require('./src/services/asset_path'); module.exports = { mode: 'production', From 5b30bae5b27675301df2cc9b409076be8d603c7a Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 21:11:03 +0300 Subject: [PATCH 22/29] bin: Adapt copy-trilium to TypeScript changes --- bin/copy-trilium.sh | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/bin/copy-trilium.sh b/bin/copy-trilium.sh index 9fdfe0293..8affb7be3 100755 --- a/bin/copy-trilium.sh +++ b/bin/copy-trilium.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +shopt -s globstar +set -e + if [[ $# -eq 0 ]] ; then echo "Missing argument of target directory" exit 1 @@ -9,25 +12,36 @@ if ! [[ $(which npm) ]]; then exit 1 fi -n exec 18.18.2 npm run webpack || npm run webpack +# Trigger the TypeScript build +echo TypeScript build start +npx tsc +echo TypeScript build finished +# Copy the TypeScript artifacts DIR="$1" - rm -rf "$DIR" mkdir -pv "$DIR" +echo Webpack start +npm run webpack +echo Webpack finish + echo "Copying Trilium to build directory $DIR" for d in 'images' 'libraries' 'src' 'db'; do cp -r "$d" "$DIR"/ done -for f in 'package.json' 'package-lock.json' 'README.md' 'LICENSE' 'config-sample.ini' 'electron.js'; do + +for f in 'package.json' 'package-lock.json' 'README.md' 'LICENSE' 'config-sample.ini'; do cp "$f" "$DIR"/ done -cp webpack-* "$DIR"/ # here warning because there is no 'webpack-*', but webpack.config.js only + +script_dir=$(realpath $(dirname $0)) +cp -Rv "$script_dir/../build/src" "$DIR" +cp "$script_dir/../build/electron.js" "$DIR" # run in subshell (so we return to original dir) -(cd $DIR && n exec 18.18.2 npm install --only=prod) +(cd $DIR && npm install --only=prod) if [[ -d "$DIR"/node_modules ]]; then # cleanup of useless files in dependencies From 7c43a49046acbee943509f4ba2a477fe87758f6a Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 21:35:34 +0300 Subject: [PATCH 23/29] ci: Add workflow for building macOS x86_64 --- .github/workflows/dev.yml | 2 ++ .github/workflows/main.yml | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 392046d2c..58e21d36a 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -1,6 +1,8 @@ name: Dev on: push: + branches_ignore: + - 'feature/fix_build_scripts' jobs: build_docker: name: Build Docker image diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 747432396..5498558be 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,10 +3,27 @@ on: push: branches: - 'develop' + - 'feature/fix_build_scripts' env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} jobs: + build_macos_64: + name: Build macOS x86_64 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up node & dependencies + uses: actions/setup-node@v4 + with: + node-version: 18 + cache: "npm" + - run: npm ci + - run: ./bin/build-mac-x64.sh + - uses: actions/upload-artifact@v4 + with: + name: trilium-mac-x64.zip + path: dist/trilium-mac-x64*.zip build_docker: name: Build Docker image runs-on: ubuntu-latest From e6ff99198f988247088d1e5ea3089ff4ad5b356c Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 22:10:57 +0300 Subject: [PATCH 24/29] ci: Fix docker build after macOS integration --- .dockerignore | 3 ++- Dockerfile | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index 941905c66..64bcb6983 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,4 +6,5 @@ /npm-debug.log node_modules -src/**/*.ts \ No newline at end of file +src/**/*.ts +!src/services/asset_path.ts \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index dfeb4a0d7..78f058529 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,8 @@ RUN set -x \ && npm prune --omit=dev \ && cp src/public/app/share.js src/public/app-dist/. \ && cp -r src/public/app/doc_notes src/public/app-dist/. \ - && rm -rf src/public/app + && rm -rf src/public/app \ + && rm src/services/asset_path.ts # Some setup tools need to be kept RUN apk add --no-cache su-exec shadow From 99ea741275a3c06e715249b4ba625a64c803e231 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 22:52:28 +0300 Subject: [PATCH 25/29] ci: Add build Linux --- .github/workflows/main.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5498558be..eb6f54821 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,7 +8,7 @@ env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} jobs: - build_macos_64: + build_darwin-x64: name: Build macOS x86_64 runs-on: ubuntu-latest steps: @@ -24,6 +24,26 @@ jobs: with: name: trilium-mac-x64.zip path: dist/trilium-mac-x64*.zip + build_linux-x64: + name: Build Linux x86_64 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up node & dependencies + uses: actions/setup-node@v4 + with: + node-version: 18 + cache: "npm" + - run: npm ci + - run: ./bin/build-linux-x64.sh + - uses: actions/upload-artifact@v4 + with: + name: trilium-linux-x64.tar.xz + path: dist/trilium-linux-x64-*.tar.xz + - uses: actions/upload-artifact@v4 + with: + name: trilium_amd64.deb + path: dist/trilium_*.deb build_docker: name: Build Docker image runs-on: ubuntu-latest From 7aed6d4ad47fd1a220ca6dd8b7fdf636a41639d2 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 23:16:00 +0300 Subject: [PATCH 26/29] ci: Add build Linux Server --- .github/workflows/main.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index eb6f54821..d826bf63f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,6 +44,22 @@ jobs: with: name: trilium_amd64.deb path: dist/trilium_*.deb + build_linux_server-x64: + name: Build Linux Server x86_64 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up node & dependencies + uses: actions/setup-node@v4 + with: + node-version: 18 + cache: "npm" + - run: npm ci + - run: ./bin/build-server.sh + - uses: actions/upload-artifact@v4 + with: + name: trilium-linux-x64-server-x64.tar.xz + path: dist/trilium-linux-x64-server-*.tar.xz build_docker: name: Build Docker image runs-on: ubuntu-latest From 998db5ce95aead5427e51b1a9954c567a06c9dd0 Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sat, 13 Jul 2024 23:23:37 +0300 Subject: [PATCH 27/29] ci: Add build Windows --- .github/workflows/main.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d826bf63f..ba4f6052a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,8 +58,24 @@ jobs: - run: ./bin/build-server.sh - uses: actions/upload-artifact@v4 with: - name: trilium-linux-x64-server-x64.tar.xz + name: trilium-linux-x64-server.tar.xz path: dist/trilium-linux-x64-server-*.tar.xz + build_windows-x64: + name: Build Windows x86_64 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up node & dependencies + uses: actions/setup-node@v4 + with: + node-version: 18 + cache: "npm" + - run: npm ci + - run: ./bin/build-win-x64.sh + - uses: actions/upload-artifact@v4 + with: + name: trilium-windows-x64.zip + path: dist/trilium-windows-x64-*.zip build_docker: name: Build Docker image runs-on: ubuntu-latest From 3f5e5fd840cc035ccc55717e8ea2f3649a9c579c Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sun, 14 Jul 2024 00:28:30 +0300 Subject: [PATCH 28/29] ci: Set up Wine --- .github/workflows/main.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ba4f6052a..6f1269a70 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -64,6 +64,13 @@ jobs: name: Build Windows x86_64 runs-on: ubuntu-latest steps: + - name: Set up Wine + run: | + sudo dpkg --add-architecture i386 + wget -qO - https://dl.winehq.org/wine-builds/winehq.key | sudo apt-key add - + sudo add-apt-repository ppa:cybermax-dexter/sdl2-backport + sudo apt-add-repository "deb https://dl.winehq.org/wine-builds/ubuntu $(lsb_release -cs) main" + sudo apt install --install-recommends winehq-stable - uses: actions/checkout@v4 - name: Set up node & dependencies uses: actions/setup-node@v4 From 75ebe24abbc84af0a6fb47e23bcc298d129dd97e Mon Sep 17 00:00:00 2001 From: Elian Doran Date: Sun, 14 Jul 2024 00:43:04 +0300 Subject: [PATCH 29/29] ci: Remove temporary branch config --- .github/workflows/dev.yml | 2 -- .github/workflows/main.yml | 1 - 2 files changed, 3 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 58e21d36a..392046d2c 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -1,8 +1,6 @@ name: Dev on: push: - branches_ignore: - - 'feature/fix_build_scripts' jobs: build_docker: name: Build Docker image diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6f1269a70..529e685e9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,7 +3,6 @@ on: push: branches: - 'develop' - - 'feature/fix_build_scripts' env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }}