From cefc402263cd1091fb10cc67dc22f6b6ac4449e5 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Mon, 17 Feb 2025 22:09:27 +0100 Subject: [PATCH 01/33] build: add separate tsconfig.build.json this prevents tsc from unnecessarily transpiling the frontend part as well: previously it was transpiled by tsc, but the files got discarded and replaced by the files built by webpack. speeds up tsc command a bit as well: from 14 seconds to ~8 secs --- package.json | 2 +- tsconfig.build.json | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 tsconfig.build.json diff --git a/package.json b/package.json index 6cddb2e32..58e7f6840 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "docs:build-frontend": "rimraf ./docs/frontend_api && jsdoc -c jsdoc-conf.json -d ./docs/frontend_api src/public/app/entities/*.js src/public/app/services/frontend_script_api.js src/public/app/widgets/basic_widget.js src/public/app/widgets/note_context_aware_widget.js src/public/app/widgets/right_panel_widget.js", "docs:build": "npm run docs:build-backend && npm run docs:build-frontend", "build:webpack": "tsx node_modules/webpack/bin/webpack.js -c webpack.config.ts", - "build:prepare-dist": "npm run build:webpack && rimraf ./dist && tsc && tsx ./bin/copy-dist.ts", + "build:prepare-dist": "npm run build:webpack && rimraf ./dist && tsc -p tsconfig.build.json && tsx ./bin/copy-dist.ts", "test": "npm run client:test && npm run server:test", "server:test": "cross-env TRILIUM_ENV=dev TRILIUM_DATA_DIR=./integration-tests/db TRILIUM_INTEGRATION_TEST=memory vitest", "server:coverage": "cross-env TRILIUM_ENV=dev TRILIUM_DATA_DIR=./integration-tests/db TRILIUM_INTEGRATION_TEST=memory vitest --coverage", diff --git a/tsconfig.build.json b/tsconfig.build.json new file mode 100644 index 000000000..24a9aa544 --- /dev/null +++ b/tsconfig.build.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "module": "NodeNext", + "declaration": false, + "sourceMap": true, + "outDir": "./build", + "strict": true, + "noImplicitAny": true, + "resolveJsonModule": true, + "allowJs": true, + "lib": ["ES2023"], + "downlevelIteration": true, + "skipLibCheck": true, + "esModuleInterop": true, + "verbatimModuleSyntax": true + }, + "include": ["./src/**/*.[jt]s", "./*.ts"], + "exclude": [ + "./node_modules/**/*", + "./spec-es6/**/*.ts", + "./spec/**/*", + "./**/*.spec.ts", + "./src/public/**/*", + "./*.config.[jt]s", + ], + "files": ["src/types.d.ts"] +} From 745b294ca1c59a7304620060c0021da53871d799 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Mon, 17 Feb 2025 23:01:55 +0100 Subject: [PATCH 02/33] build(webpack): output webpack build in build folder output the bundled files directly in the build folder a) keeps the src folder clean from build output b) it saves us some "manual" copying work --- webpack.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webpack.config.ts b/webpack.config.ts index bd16bbaec..3b2e45353 100644 --- a/webpack.config.ts +++ b/webpack.config.ts @@ -19,7 +19,7 @@ const config: Configuration = { }, output: { publicPath: `${assetPath}/app-dist/`, - path: path.resolve(rootDir, "src/public/app-dist"), + path: path.resolve(rootDir, "build/src/public/app-dist"), filename: "[name].js" }, plugins: [ From 2be9389f3bb2b8eda1fae349eddbf4ca9ee4a8c6 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Mon, 17 Feb 2025 23:07:54 +0100 Subject: [PATCH 03/33] build(webpack): change to nosources-source-map as per https://webpack.js.org/configuration/devtool/#production serving the `source-map` file to "normal" users seems to be not recommended, so instead let's go with `nosources-source-map`: a) this drastically reduces app-dist folder size from 20MB down to 8.7MB b) it still allows for stack traces --- webpack.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webpack.config.ts b/webpack.config.ts index 3b2e45353..82439ea6d 100644 --- a/webpack.config.ts +++ b/webpack.config.ts @@ -75,7 +75,7 @@ const config: Configuration = { ".mjs": [".mjs", ".mts"] } }, - devtool: "source-map", + devtool: "nosources-source-map", target: "electron-renderer" }; From e0c0086eb38e023c5ed2654a7d47c64c31475452 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Mon, 17 Feb 2025 23:30:22 +0100 Subject: [PATCH 04/33] build(tsconfig): fix glob for build config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit not sure why, but seems like it doesn't like `[jt]s` – which causes it to skip certain .d.ts files, making tsc fail --- tsconfig.build.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tsconfig.build.json b/tsconfig.build.json index 24a9aa544..03a0e159b 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -14,14 +14,14 @@ "esModuleInterop": true, "verbatimModuleSyntax": true }, - "include": ["./src/**/*.[jt]s", "./*.ts"], + "include": ["./src/**/*.ts", "./src/**/*.js", "./*.ts"], "exclude": [ "./node_modules/**/*", "./spec-es6/**/*.ts", "./spec/**/*", "./**/*.spec.ts", "./src/public/**/*", - "./*.config.[jt]s", + "./*.config.ts", ], "files": ["src/types.d.ts"] } From 3a87d710732116488c0c333928dcf61ee3f076bf Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Mon, 17 Feb 2025 23:44:37 +0100 Subject: [PATCH 05/33] build(scripts): add build:ts and update build:prepare-dist since we build into the build folder -> we should also clean the folder before building as well also it makes sense to run tsc first, as it runs faster, so if there's any TS errors, we will have a faster failing build --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 58e7f6840..6c9704365 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,8 @@ "docs:build-frontend": "rimraf ./docs/frontend_api && jsdoc -c jsdoc-conf.json -d ./docs/frontend_api src/public/app/entities/*.js src/public/app/services/frontend_script_api.js src/public/app/widgets/basic_widget.js src/public/app/widgets/note_context_aware_widget.js src/public/app/widgets/right_panel_widget.js", "docs:build": "npm run docs:build-backend && npm run docs:build-frontend", "build:webpack": "tsx node_modules/webpack/bin/webpack.js -c webpack.config.ts", - "build:prepare-dist": "npm run build:webpack && rimraf ./dist && tsc -p tsconfig.build.json && tsx ./bin/copy-dist.ts", + "build:ts": "tsc -p tsconfig.build.json", + "build:prepare-dist": "rimraf ./dist && rimraf ./build && npm run build:ts && npm run build:webpack && tsx ./bin/copy-dist.ts", "test": "npm run client:test && npm run server:test", "server:test": "cross-env TRILIUM_ENV=dev TRILIUM_DATA_DIR=./integration-tests/db TRILIUM_INTEGRATION_TEST=memory vitest", "server:coverage": "cross-env TRILIUM_ENV=dev TRILIUM_DATA_DIR=./integration-tests/db TRILIUM_INTEGRATION_TEST=memory vitest --coverage", From d0bb5f97688ee2060c4d1967fc1ec97ebb8850ec Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Fri, 21 Feb 2025 20:02:10 +0100 Subject: [PATCH 06/33] build(scripts): add webpack progress flag --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6c9704365..d8f0a3215 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "docs:build-backend": "rimraf ./docs/backend_api && typedoc ./docs/backend_api src/becca/entities/*.ts src/services/backend_script_api.ts src/services/sql.ts", "docs:build-frontend": "rimraf ./docs/frontend_api && jsdoc -c jsdoc-conf.json -d ./docs/frontend_api src/public/app/entities/*.js src/public/app/services/frontend_script_api.js src/public/app/widgets/basic_widget.js src/public/app/widgets/note_context_aware_widget.js src/public/app/widgets/right_panel_widget.js", "docs:build": "npm run docs:build-backend && npm run docs:build-frontend", - "build:webpack": "tsx node_modules/webpack/bin/webpack.js -c webpack.config.ts", + "build:webpack": "tsx node_modules/webpack/bin/webpack.js -c webpack.config.ts --progress", "build:ts": "tsc -p tsconfig.build.json", "build:prepare-dist": "rimraf ./dist && rimraf ./build && npm run build:ts && npm run build:webpack && tsx ./bin/copy-dist.ts", "test": "npm run client:test && npm run server:test", From 6c4a1732ad5ec8b33e61cb8825aaacc782c9518e Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Sat, 22 Feb 2025 13:25:36 +0100 Subject: [PATCH 07/33] build(copy-trilium): use `npm run build:ts` --- bin/copy-trilium.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/copy-trilium.sh b/bin/copy-trilium.sh index e1d0e197f..f62b180a0 100755 --- a/bin/copy-trilium.sh +++ b/bin/copy-trilium.sh @@ -14,7 +14,7 @@ fi # Trigger the TypeScript build echo TypeScript build start -npx tsc +npm run build:ts echo TypeScript build finished # Copy the TypeScript artifacts From 3e307f4711808ea8d3c9bc7d5cb2044a581ddeb1 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Sun, 23 Feb 2025 17:49:37 +0100 Subject: [PATCH 08/33] build(scripts): add "build:clean" script for removing dist and build folders --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index d8f0a3215..acd61a56b 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,8 @@ "docs:build": "npm run docs:build-backend && npm run docs:build-frontend", "build:webpack": "tsx node_modules/webpack/bin/webpack.js -c webpack.config.ts --progress", "build:ts": "tsc -p tsconfig.build.json", - "build:prepare-dist": "rimraf ./dist && rimraf ./build && npm run build:ts && npm run build:webpack && tsx ./bin/copy-dist.ts", + "build:clean": "rimraf ./dist ./build", + "build:prepare-dist": "npm run build:clean && npm run build:ts && npm run build:webpack && tsx ./bin/copy-dist.ts", "test": "npm run client:test && npm run server:test", "server:test": "cross-env TRILIUM_ENV=dev TRILIUM_DATA_DIR=./integration-tests/db TRILIUM_INTEGRATION_TEST=memory vitest", "server:coverage": "cross-env TRILIUM_ENV=dev TRILIUM_DATA_DIR=./integration-tests/db TRILIUM_INTEGRATION_TEST=memory vitest --coverage", From e483cbca3cb4e0ae7bf9f172d9815472798ec1d1 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Sun, 23 Feb 2025 18:16:33 +0100 Subject: [PATCH 09/33] build(tsconfig): remove unnecessary exclude lines these folder are already "excluded" implicitly, since we only include "./src" folder --- tsconfig.build.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/tsconfig.build.json b/tsconfig.build.json index 03a0e159b..1987c96c0 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -16,9 +16,6 @@ }, "include": ["./src/**/*.ts", "./src/**/*.js", "./*.ts"], "exclude": [ - "./node_modules/**/*", - "./spec-es6/**/*.ts", - "./spec/**/*", "./**/*.spec.ts", "./src/public/**/*", "./*.config.ts", From 60a2b56636aee913faf47a02efd824ca4e162ac9 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Sun, 23 Feb 2025 18:37:29 +0100 Subject: [PATCH 10/33] chore(copy-dist): removee unnecesary copy of tsconfig.webpack.json webpack bundling already ran before this script, so there is no need to copy this file over --- bin/copy-dist.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/copy-dist.ts b/bin/copy-dist.ts index 2a2b75d56..6dae9b3dc 100644 --- a/bin/copy-dist.ts +++ b/bin/copy-dist.ts @@ -31,7 +31,6 @@ const copy = async () => { const filesToCopy = [ "config-sample.ini", - "tsconfig.webpack.json", "./src/etapi/etapi.openapi.yaml", "./src/routes/api/openapi.json" ]; From 135101f57ba183971bbdf1fb315380378c653145 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Mon, 24 Feb 2025 20:27:06 +0100 Subject: [PATCH 11/33] build(copy-dist): do not copy build folder into src folder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit stop the build folder from being copied into the dist/src subfolder → there is no sense in doing that → the contents of the build folder are corretly copied previously already (see line 26ff) --- bin/copy-dist.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/copy-dist.ts b/bin/copy-dist.ts index 6dae9b3dc..9eaf2cbc2 100644 --- a/bin/copy-dist.ts +++ b/bin/copy-dist.ts @@ -45,7 +45,7 @@ const copy = async () => { await fs.copy(dir, path.join(DEST_DIR, dir)); } - const srcDirsToCopy = ["./src/public", "./src/views", "./build"]; + const srcDirsToCopy = ["./src/public", "./src/views"]; for (const dir of srcDirsToCopy) { log(`Copying ${dir}`); await fs.copy(dir, path.join(DEST_DIR_SRC, path.basename(dir))); From af5e4ee3b57ca583f49710133e52735d97eb9568 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Wed, 26 Feb 2025 21:24:00 +0100 Subject: [PATCH 12/33] build(copy-dist): copy over required folders/files from "public" folder --- bin/copy-dist.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/bin/copy-dist.ts b/bin/copy-dist.ts index 9eaf2cbc2..4ae25c681 100644 --- a/bin/copy-dist.ts +++ b/bin/copy-dist.ts @@ -32,14 +32,25 @@ const copy = async () => { const filesToCopy = [ "config-sample.ini", "./src/etapi/etapi.openapi.yaml", - "./src/routes/api/openapi.json" + "./src/routes/api/openapi.json", + "./src/public/icon.png", + "./src/public/manifest.webmanifest", + "./src/public/robots.txt" ]; for (const file of filesToCopy) { log(`Copying ${file}`); await fs.copy(file, path.join(DEST_DIR, file)); } - const dirsToCopy = ["images", "libraries", "translations", "db"]; + const dirsToCopy = [ + "images", + "libraries", + "translations", + "db", + "src/public/fonts", + "src/public/stylesheets", + "src/public/translations" + ]; for (const dir of dirsToCopy) { log(`Copying ${dir}`); await fs.copy(dir, path.join(DEST_DIR, dir)); From 24382d8176348868c4523e0e92cc94453630c75a Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Wed, 26 Feb 2025 21:43:48 +0100 Subject: [PATCH 13/33] build(copy-dist): avoid copying over the app dir into dist we have the bundled "app-dist" already in the "dist", copying over the original unbundled "app" folder serves no benefit here --- bin/copy-dist.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/copy-dist.ts b/bin/copy-dist.ts index 4ae25c681..5b861ca9e 100644 --- a/bin/copy-dist.ts +++ b/bin/copy-dist.ts @@ -56,7 +56,7 @@ const copy = async () => { await fs.copy(dir, path.join(DEST_DIR, dir)); } - const srcDirsToCopy = ["./src/public", "./src/views"]; + const srcDirsToCopy = ["./src/views"]; for (const dir of srcDirsToCopy) { log(`Copying ${dir}`); await fs.copy(dir, path.join(DEST_DIR_SRC, path.basename(dir))); From dbc2df0820e7f18f66bdc59d1c98184ea58ed8c1 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Wed, 26 Feb 2025 21:54:45 +0100 Subject: [PATCH 14/33] build(copy-dist): consolidate folder copying the "srcDirsToCopy" block is useless now, we can just use the previous dirsToCopy to achieve the exact same thing --- bin/copy-dist.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/bin/copy-dist.ts b/bin/copy-dist.ts index 5b861ca9e..208825b46 100644 --- a/bin/copy-dist.ts +++ b/bin/copy-dist.ts @@ -49,19 +49,14 @@ const copy = async () => { "db", "src/public/fonts", "src/public/stylesheets", - "src/public/translations" + "src/public/translations", + "src/views/" ]; for (const dir of dirsToCopy) { log(`Copying ${dir}`); await fs.copy(dir, path.join(DEST_DIR, dir)); } - const srcDirsToCopy = ["./src/views"]; - for (const dir of srcDirsToCopy) { - log(`Copying ${dir}`); - await fs.copy(dir, path.join(DEST_DIR_SRC, path.basename(dir))); - } - /** * Directories to be copied relative to the project root into /src/public/app-dist. */ From 9ac451e2b2570d12efd66fa6a7bde33ff8c5b288 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Thu, 27 Feb 2025 08:01:07 +0100 Subject: [PATCH 15/33] build(copy-dist): consolidate files and folder copying into one asset copying job there's no benefit in having them split up like before --- bin/copy-dist.ts | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/bin/copy-dist.ts b/bin/copy-dist.ts index 208825b46..c7bd8c809 100644 --- a/bin/copy-dist.ts +++ b/bin/copy-dist.ts @@ -29,32 +29,26 @@ const copy = async () => { fs.copySync(path.join("build", srcFile), destFile, { recursive: true }); } - const filesToCopy = [ - "config-sample.ini", + const assetsToCopy = new Set([ + "./images", + "./libraries", + "./translations", + "./db", + "./config-sample.ini", + "./src/views/", "./src/etapi/etapi.openapi.yaml", "./src/routes/api/openapi.json", "./src/public/icon.png", "./src/public/manifest.webmanifest", - "./src/public/robots.txt" - ]; - for (const file of filesToCopy) { - log(`Copying ${file}`); - await fs.copy(file, path.join(DEST_DIR, file)); - } + "./src/public/robots.txt", + "./src/public/fonts", + "./src/public/stylesheets", + "./src/public/translations" + ]); - const dirsToCopy = [ - "images", - "libraries", - "translations", - "db", - "src/public/fonts", - "src/public/stylesheets", - "src/public/translations", - "src/views/" - ]; - for (const dir of dirsToCopy) { - log(`Copying ${dir}`); - await fs.copy(dir, path.join(DEST_DIR, dir)); + for (const asset of assetsToCopy) { + log(`Copying ${asset}`); + await fs.copy(asset, path.join(DEST_DIR, asset)); } /** From ac3e96291aad41ac2f21abc776667f78f79aea07 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Thu, 27 Feb 2025 08:32:08 +0100 Subject: [PATCH 16/33] build(copy-dist): simplify "build" copying there's no need to read the folder structure and then copy each single file in a loop => just copy the whole folder and be done with it :-) --- bin/copy-dist.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/bin/copy-dist.ts b/bin/copy-dist.ts index c7bd8c809..45514e900 100644 --- a/bin/copy-dist.ts +++ b/bin/copy-dist.ts @@ -23,11 +23,8 @@ async function copyNodeModuleFileOrFolder(source: string) { } const copy = async () => { - for (const srcFile of fs.readdirSync("build")) { - const destFile = path.join(DEST_DIR, path.basename(srcFile)); - log(`Copying source ${srcFile} -> ${destFile}.`); - fs.copySync(path.join("build", srcFile), destFile, { recursive: true }); - } + log(`Copying build into dist folder.`); + fs.copySync("./build", DEST_DIR); const assetsToCopy = new Set([ "./images", From dbeae62709c894390279b51f7ce93da22d64fc24 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Thu, 27 Feb 2025 08:53:02 +0100 Subject: [PATCH 17/33] build(copy-dist): simplify "copyNodeModuleFileOrFolder" copying there's no benefit from stripping "node_modules/" from the string, to later add it again using the `DEST_DIR_NODE_MODULES` constant => just copy directly into the `DEST_DIR` folder and preserver the `node_modules` part in the path --- bin/copy-dist.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bin/copy-dist.ts b/bin/copy-dist.ts index 45514e900..e5f8b6603 100644 --- a/bin/copy-dist.ts +++ b/bin/copy-dist.ts @@ -14,9 +14,7 @@ function log(...args: any[]) { } async function copyNodeModuleFileOrFolder(source: string) { - const adjustedSource = source.substring(13); - const destination = path.join(DEST_DIR_NODE_MODULES, adjustedSource); - + const destination = path.join(DEST_DIR, source); log(`Copying ${source} to ${destination}`); await fs.ensureDir(path.dirname(destination)); await fs.copy(source, destination); From 37f1525d0e688dcc2d59ceaf7218b81f8c01683c Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Thu, 27 Feb 2025 08:53:38 +0100 Subject: [PATCH 18/33] build(copy-dist): remove unused paths --- bin/copy-dist.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/bin/copy-dist.ts b/bin/copy-dist.ts index e5f8b6603..4c69900cd 100644 --- a/bin/copy-dist.ts +++ b/bin/copy-dist.ts @@ -2,8 +2,6 @@ import fs from "fs-extra"; import path from "path"; const DEST_DIR = "./dist"; -const DEST_DIR_SRC = path.join(DEST_DIR, "src"); -const DEST_DIR_NODE_MODULES = path.join(DEST_DIR, "node_modules"); const VERBOSE = process.env.VERBOSE; From a816abb3723a8524fa0513f48776103b040c4eb1 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Thu, 27 Feb 2025 08:59:12 +0100 Subject: [PATCH 19/33] build(copy-dist): use sync copying since this is a "standalone" script we are running and no other JS scritps are running "in the background", there's no real benefit for async here. --- bin/copy-dist.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bin/copy-dist.ts b/bin/copy-dist.ts index 4c69900cd..eb721e850 100644 --- a/bin/copy-dist.ts +++ b/bin/copy-dist.ts @@ -11,11 +11,11 @@ function log(...args: any[]) { } } -async function copyNodeModuleFileOrFolder(source: string) { +function copyNodeModuleFileOrFolder(source: string) { const destination = path.join(DEST_DIR, source); log(`Copying ${source} to ${destination}`); - await fs.ensureDir(path.dirname(destination)); - await fs.copy(source, destination); + fs.ensureDirSync(path.dirname(destination)); + fs.copySync(source, destination); } const copy = async () => { @@ -41,7 +41,7 @@ const copy = async () => { for (const asset of assetsToCopy) { log(`Copying ${asset}`); - await fs.copy(asset, path.join(DEST_DIR, asset)); + fs.copySync(asset, path.join(DEST_DIR, asset)); } /** @@ -50,7 +50,7 @@ const copy = async () => { const publicDirsToCopy = ["./src/public/app/doc_notes"]; const PUBLIC_DIR = path.join(DEST_DIR, "src", "public", "app-dist"); for (const dir of publicDirsToCopy) { - await fs.copy(dir, path.join(PUBLIC_DIR, path.basename(dir))); + fs.copySync(dir, path.join(PUBLIC_DIR, path.basename(dir))); } const nodeModulesFile = [ @@ -66,7 +66,7 @@ const copy = async () => { ]; for (const file of nodeModulesFile) { - await copyNodeModuleFileOrFolder(file); + copyNodeModuleFileOrFolder(file); } const nodeModulesFolder = [ @@ -99,7 +99,7 @@ const copy = async () => { ]; for (const folder of nodeModulesFolder) { - await copyNodeModuleFileOrFolder(folder); + copyNodeModuleFileOrFolder(folder); } }; From d75cf8c11e429a762d821d41107190264052cdc6 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Thu, 27 Feb 2025 09:02:01 +0100 Subject: [PATCH 20/33] build(copy-dist): consolidate nodeModules copying into a single loop --- bin/copy-dist.ts | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/bin/copy-dist.ts b/bin/copy-dist.ts index eb721e850..9f52f8a5d 100644 --- a/bin/copy-dist.ts +++ b/bin/copy-dist.ts @@ -53,7 +53,7 @@ const copy = async () => { fs.copySync(dir, path.join(PUBLIC_DIR, path.basename(dir))); } - const nodeModulesFile = [ + const nodeModulesFile = new Set([ "node_modules/react/umd/react.production.min.js", "node_modules/react/umd/react.development.js", "node_modules/react-dom/umd/react-dom.production.min.js", @@ -63,13 +63,9 @@ const copy = async () => { "node_modules/katex/dist/contrib/auto-render.min.js", "node_modules/@highlightjs/cdn-assets/highlight.min.js", "node_modules/@mind-elixir/node-menu/dist/node-menu.umd.cjs" - ]; + ]); - for (const file of nodeModulesFile) { - copyNodeModuleFileOrFolder(file); - } - - const nodeModulesFolder = [ + const nodeModulesFolder = new Set([ "node_modules/@excalidraw/excalidraw/dist/", "node_modules/katex/dist/", "node_modules/dayjs/", @@ -96,10 +92,12 @@ const copy = async () => { "node_modules/@highlightjs/cdn-assets/languages", "node_modules/@highlightjs/cdn-assets/styles", "node_modules/leaflet/dist" - ]; + ]); - for (const folder of nodeModulesFolder) { - copyNodeModuleFileOrFolder(folder); + + + for (const nodeModuleItem of [...nodeModulesFile, ...nodeModulesFolder]) { + copyNodeModuleFileOrFolder(nodeModuleItem); } }; From 3032156b45dd611e1328ecbce66c10fcf8f6bd67 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Thu, 27 Feb 2025 09:10:27 +0100 Subject: [PATCH 21/33] build(copy-dist): execute code in try/catch -> get rid of function since we don't export this anywhere, might as well just call the steps directly --- bin/copy-dist.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bin/copy-dist.ts b/bin/copy-dist.ts index 9f52f8a5d..f752782c9 100644 --- a/bin/copy-dist.ts +++ b/bin/copy-dist.ts @@ -18,7 +18,8 @@ function copyNodeModuleFileOrFolder(source: string) { fs.copySync(source, destination); } -const copy = async () => { +try { + log(`Copying build into dist folder.`); fs.copySync("./build", DEST_DIR); @@ -99,8 +100,8 @@ const copy = async () => { for (const nodeModuleItem of [...nodeModulesFile, ...nodeModulesFolder]) { copyNodeModuleFileOrFolder(nodeModuleItem); } -}; + console.log("Copying complete!") -copy() - .then(() => console.log("Copying complete!")) - .catch((err) => console.error("Error during copy:", err)); +} catch(err) { + console.error("Error during copy:", err) +} \ No newline at end of file From 5e289ea12dc840dc5564b497622b57051949f380 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Mon, 3 Mar 2025 08:40:43 +0100 Subject: [PATCH 22/33] build: get rid of intermediary "build" folder -> use "dist" directly --- tsconfig.build.json | 2 +- tsconfig.json | 2 +- tsconfig.webpack.json | 2 +- webpack.config.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tsconfig.build.json b/tsconfig.build.json index 1987c96c0..4d4ffd342 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -3,7 +3,7 @@ "module": "NodeNext", "declaration": false, "sourceMap": true, - "outDir": "./build", + "outDir": "./dist", "strict": true, "noImplicitAny": true, "resolveJsonModule": true, diff --git a/tsconfig.json b/tsconfig.json index c1f3ef48b..d94fcef82 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,7 @@ "module": "NodeNext", "declaration": false, "sourceMap": true, - "outDir": "./build", + "outDir": "./dist", "strict": true, "noImplicitAny": true, "resolveJsonModule": true, diff --git a/tsconfig.webpack.json b/tsconfig.webpack.json index ed622818b..cbfe8cbdb 100644 --- a/tsconfig.webpack.json +++ b/tsconfig.webpack.json @@ -3,7 +3,7 @@ "module": "NodeNext", "declaration": false, "sourceMap": true, - "outDir": "./build", + "outDir": "./dist", "strict": true, "noImplicitAny": true, "resolveJsonModule": true, diff --git a/webpack.config.ts b/webpack.config.ts index 82439ea6d..207ff2fcf 100644 --- a/webpack.config.ts +++ b/webpack.config.ts @@ -19,7 +19,7 @@ const config: Configuration = { }, output: { publicPath: `${assetPath}/app-dist/`, - path: path.resolve(rootDir, "build/src/public/app-dist"), + path: path.resolve(rootDir, "dist/src/public/app-dist"), filename: "[name].js" }, plugins: [ From e389592017b883beafb3bbdeb15ac86775c07fc6 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Mon, 3 Mar 2025 09:12:50 +0100 Subject: [PATCH 23/33] build(copy-dist): copying build folder is not required anymore TS and Webpack build into the dist folder directly now --- bin/copy-dist.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/bin/copy-dist.ts b/bin/copy-dist.ts index f752782c9..c8911022a 100644 --- a/bin/copy-dist.ts +++ b/bin/copy-dist.ts @@ -20,9 +20,6 @@ function copyNodeModuleFileOrFolder(source: string) { try { - log(`Copying build into dist folder.`); - fs.copySync("./build", DEST_DIR); - const assetsToCopy = new Set([ "./images", "./libraries", From bb7a4f9bc39ad44bcccf8d17f7745ed541e0c9b2 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Mon, 3 Mar 2025 09:16:15 +0100 Subject: [PATCH 24/33] build(Docker): comment out seemingly useless installation of build dependencies --- Dockerfile | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 365e4d07f..cd274a500 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,17 +2,18 @@ FROM node:22.14.0-bullseye-slim AS builder # Configure build dependencies in a single layer -RUN apt-get update && apt-get install -y --no-install-recommends \ - autoconf \ - automake \ - g++ \ - gcc \ - libtool \ - make \ - nasm \ - libpng-dev \ - python3 \ - && rm -rf /var/lib/apt/lists/* +# TriliumNextTODO: These don't seem to be required at all +# RUN apt-get update && apt-get install -y --no-install-recommends \ +# autoconf \ +# automake \ +# g++ \ +# gcc \ +# libtool \ +# make \ +# nasm \ +# libpng-dev \ +# python3 \ +# && rm -rf /var/lib/apt/lists/* WORKDIR /usr/src/app From 2973d38db0f3ef9cfdb5ab8d5936ed3015d20a63 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Mon, 3 Mar 2025 09:28:23 +0100 Subject: [PATCH 25/33] build(Docker): move server-package.json preparation into Dockerfile --- .github/workflows/dev.yml | 5 ----- .github/workflows/main-docker.yml | 5 ----- Dockerfile | 4 ++-- Dockerfile.alpine | 3 +-- bin/build-docker.sh | 2 -- 5 files changed, 3 insertions(+), 16 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index d2ad631b0..8d50a4640 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -52,8 +52,6 @@ jobs: - run: npm ci - name: Run the TypeScript build run: npx tsc - - name: Create server-package.json - run: cat package.json | grep -v electron > server-package.json - uses: docker/setup-buildx-action@v3 - uses: docker/build-push-action@v6 with: @@ -93,9 +91,6 @@ jobs: - name: Run the TypeScript build run: npx tsc - - name: Create server-package.json - run: cat package.json | grep -v electron > server-package.json - - name: Build and export to Docker uses: docker/build-push-action@v6 with: diff --git a/.github/workflows/main-docker.yml b/.github/workflows/main-docker.yml index 0c1be531a..d8331184f 100644 --- a/.github/workflows/main-docker.yml +++ b/.github/workflows/main-docker.yml @@ -57,9 +57,6 @@ jobs: - name: Run the TypeScript build run: npx tsc - - name: Create server-package.json - run: cat package.json | grep -v electron > server-package.json - - name: Build and export to Docker uses: docker/build-push-action@v6 with: @@ -163,8 +160,6 @@ jobs: - run: npm ci - name: Run the TypeScript build run: npx tsc - - name: Create server-package.json - run: cat package.json | grep -v electron > server-package.json - name: Login to GHCR uses: docker/login-action@v3 diff --git a/Dockerfile b/Dockerfile index cd274a500..6577f2de6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,10 +19,10 @@ WORKDIR /usr/src/app # Copy only necessary files for build COPY . . -COPY server-package.json package.json # Build and cleanup in a single layer -RUN cp -R build/src/* src/. && \ +RUN sed -i "/electron/d" package.json && \ + cp -R build/src/* src/. && \ cp build/docker_healthcheck.js . && \ rm docker_healthcheck.ts && \ npm install && \ diff --git a/Dockerfile.alpine b/Dockerfile.alpine index 36d6f0b7b..2e134ab5e 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -17,10 +17,9 @@ WORKDIR /usr/src/app # Copy only necessary files for build COPY . . -COPY server-package.json package.json # Build and cleanup in a single layer -RUN cp -R build/src/* src/. && \ +RUN sed -i "/electron/d" package.json && \ cp build/docker_healthcheck.js . && \ rm docker_healthcheck.ts && \ npm install && \ diff --git a/bin/build-docker.sh b/bin/build-docker.sh index a765930db..892afcd92 100755 --- a/bin/build-docker.sh +++ b/bin/build-docker.sh @@ -5,8 +5,6 @@ set -e # Fail on any command error VERSION=`jq -r ".version" package.json` SERIES=${VERSION:0:4}-latest -cat package.json | grep -v electron > server-package.json - echo "Compiling typescript..." npx tsc From c68b0b02e47b7b4b62cd71c315b33c675d37fd32 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Wed, 5 Mar 2025 07:07:47 +0100 Subject: [PATCH 26/33] build(Docker): simplify Docker build and runtime stage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit this Dockerfile is aimed at production builds, i.e. trying to keep size as small as possible at the cost of "rebuild speed", due to missed docker cache opportunities. Build Stage: * do the complete build inside docker as oposed to the previous "hybrid", where tsc was run locally and the output got copied into the Docker build stage → you can now build this with Docker, without having to install the whole node/TS env locally * build into a "build" subfolder, for easier clean up during build stage * get rid of now unnecessary extra file/asset handling, as this is now handled by `npm run build:prepare-dist` * no `npm prune` needed here, as we delete the whole build folder anyways in the last build step Runtime stage: * move the "electron" dep removal from the builder stage to the runtime stage, before installing the dependencies * move to `npm ci` for reproducible installations – but only installing runtime deps here * get rid of now unnecessary copying commands from the builder stage, as everything is now neatly available in "/usr/src/app" --- Dockerfile | 53 ++++++++++++++++++++++------------------------------- 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6577f2de6..d4c7d205e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,49 +15,40 @@ FROM node:22.14.0-bullseye-slim AS builder # python3 \ # && rm -rf /var/lib/apt/lists/* -WORKDIR /usr/src/app +WORKDIR /usr/src/app/build # Copy only necessary files for build COPY . . # Build and cleanup in a single layer -RUN sed -i "/electron/d" package.json && \ - cp -R build/src/* src/. && \ - cp build/docker_healthcheck.js . && \ - rm docker_healthcheck.ts && \ - npm install && \ - npm run build:webpack && \ - npm prune --omit=dev && \ +RUN npm ci && \ + npm run build:prepare-dist && \ npm cache clean --force && \ - cp -r src/public/app/doc_notes src/public/app-dist/. && \ - rm -rf src/public/app/* && \ - mkdir -p src/public/app/services && \ - cp -r build/src/public/app/services/mime_type_definitions.js src/public/app/services/mime_type_definitions.js && \ - rm src/services/asset_path.ts && \ - rm -r build + mv dist/* \ + start-docker.sh \ + package-lock.json \ + /usr/src/app/ && \ + rm -rf /usr/src/app/build + +#TODO: move package-lock copying into copy-dist # Runtime stage FROM node:22.14.0-bullseye-slim -# Install only runtime dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - gosu \ - && rm -rf /var/lib/apt/lists/* && \ - rm -rf /var/cache/apt/* - WORKDIR /usr/src/app -# Copy only necessary files from builder -COPY --from=builder /usr/src/app/node_modules ./node_modules -COPY --from=builder /usr/src/app/src ./src -COPY --from=builder /usr/src/app/db ./db -COPY --from=builder /usr/src/app/docker_healthcheck.js . -COPY --from=builder /usr/src/app/start-docker.sh . -COPY --from=builder /usr/src/app/package.json . -COPY --from=builder /usr/src/app/config-sample.ini . -COPY --from=builder /usr/src/app/images ./images -COPY --from=builder /usr/src/app/translations ./translations -COPY --from=builder /usr/src/app/libraries ./libraries +# Install only runtime dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + gosu && \ + rm -rf /var/lib/apt/lists/* && \ + rm -rf /var/cache/apt/* + +COPY --from=builder /usr/src/app ./ + +RUN sed -i "/electron/d" package.json && \ + npm ci --omit=dev && \ + npm cache clean --force # Configure container EXPOSE 8080 From e9824c45680c327028b5df0559d474581ebe9223 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Wed, 5 Mar 2025 07:38:04 +0100 Subject: [PATCH 27/33] build(dockerignore): ignore unnecessary files --- .dockerignore | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/.dockerignore b/.dockerignore index 64bcb6983..786c22ff9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,10 +1,37 @@ -.git -.idea +# ignored Files +.dockerignore +.editorconfig +.git* +.prettier* +electron* +entitlements.plist +forge.config.cjs +nodemon.json +renovate.json +trilium.iml +Dockerfile +Dockerfile.* +npm-debug.log +/src/**/*.spec.ts + +# ignored folders +/.cache +/.git +/.github +/.idea +/.vscode /bin +/build /dist /docs -/npm-debug.log -node_modules +/dump-db +/e2e +/integration-tests +/spec +/test +/test-etapi +/node_modules -src/**/*.ts -!src/services/asset_path.ts \ No newline at end of file + +# exceptions +!/bin/copy-dist.ts \ No newline at end of file From 68875683af464f863a0cbbbf1449cd6677b25b8f Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Wed, 5 Mar 2025 07:41:39 +0100 Subject: [PATCH 28/33] build(Docker): get rid of apparently unused packages at build stage --- Dockerfile | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index d4c7d205e..54db47144 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,6 @@ # Build stage FROM node:22.14.0-bullseye-slim AS builder -# Configure build dependencies in a single layer -# TriliumNextTODO: These don't seem to be required at all -# RUN apt-get update && apt-get install -y --no-install-recommends \ -# autoconf \ -# automake \ -# g++ \ -# gcc \ -# libtool \ -# make \ -# nasm \ -# libpng-dev \ -# python3 \ -# && rm -rf /var/lib/apt/lists/* - WORKDIR /usr/src/app/build # Copy only necessary files for build From f544a84f6dd3b3569c7878af0dc295a91d7d02bf Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Wed, 5 Mar 2025 07:48:49 +0100 Subject: [PATCH 29/33] build(Docker): simplify Docker alpine build and runtime stage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit same changes as for the "non-alpine" Dockerfile previously commited, but adapted to Alpine. this Dockerfile is aimed at production builds, i.e. trying to keep size as small as possible at the cost of "rebuild speed", due to missed docker cache opportunities. Build Stage: * do the complete build inside docker as oposed to the previous "hybrid", where tsc was run locally and the output got copied into the Docker build stage → you can now build this with Docker, without having to install the whole node/TS env locally * build into a "build" subfolder, for easier clean up during build stage * get rid of now unnecessary extra file/asset handling, as this is now handled by `npm run build:prepare-dist` * no `npm prune` needed here, as we delete the whole build folder anyways in the last build step Runtime stage: * move the "electron" dep removal from the builder stage to the runtime stage, before installing the dependencies * move to `npm ci` for reproducible installations – but only installing runtime deps here * get rid of now unnecessary copying commands from the builder stage, as everything is now neatly available in "/usr/src/app" --- Dockerfile.alpine | 51 ++++++++++++++--------------------------------- 1 file changed, 15 insertions(+), 36 deletions(-) diff --git a/Dockerfile.alpine b/Dockerfile.alpine index 2e134ab5e..43eac8ca7 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -1,37 +1,22 @@ # Build stage FROM node:22.14.0-alpine AS builder -# Configure build dependencies -RUN apk add --no-cache --virtual .build-dependencies \ - autoconf \ - automake \ - g++ \ - gcc \ - libtool \ - make \ - nasm \ - libpng-dev \ - python3 - -WORKDIR /usr/src/app +WORKDIR /usr/src/app/build # Copy only necessary files for build COPY . . # Build and cleanup in a single layer -RUN sed -i "/electron/d" package.json && \ - cp build/docker_healthcheck.js . && \ - rm docker_healthcheck.ts && \ - npm install && \ - npm run build:webpack && \ - npm prune --omit=dev && \ +RUN npm ci && \ + npm run build:prepare-dist && \ npm cache clean --force && \ - cp -r src/public/app/doc_notes src/public/app-dist/. && \ - rm -rf src/public/app && \ - mkdir -p src/public/app/services && \ - cp -r build/src/public/app/services/mime_type_definitions.js src/public/app/services/mime_type_definitions.js && \ - rm src/services/asset_path.ts && \ - rm -r build + mv dist/* \ + start-docker.sh \ + package-lock.json \ + /usr/src/app/ && \ + rm -rf /usr/src/app/build + +#TODO: move package-lock copying into copy-dist # Runtime stage FROM node:22.14.0-alpine @@ -41,17 +26,11 @@ RUN apk add --no-cache su-exec shadow WORKDIR /usr/src/app -# Copy only necessary files from builder -COPY --from=builder /usr/src/app/node_modules ./node_modules -COPY --from=builder /usr/src/app/src ./src -COPY --from=builder /usr/src/app/db ./db -COPY --from=builder /usr/src/app/docker_healthcheck.js . -COPY --from=builder /usr/src/app/start-docker.sh . -COPY --from=builder /usr/src/app/package.json . -COPY --from=builder /usr/src/app/config-sample.ini . -COPY --from=builder /usr/src/app/images ./images -COPY --from=builder /usr/src/app/translations ./translations -COPY --from=builder /usr/src/app/libraries ./libraries +COPY --from=builder /usr/src/app ./ + +RUN sed -i "/electron/d" package.json && \ + npm ci --omit=dev && \ + npm cache clean --force # Add application user RUN adduser -s /bin/false node; exit 0 From 3553d64060764f8848c246ff35285971e9cdd788 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Wed, 5 Mar 2025 08:03:43 +0100 Subject: [PATCH 30/33] build(Docker/ci): get rid of running partial build locally this is now handled fully inside Docker. exception for "test_docker" job in "main-docker" -> it seems that one needs to be there still, since it runs Playwright tests from outside the container --- .github/workflows/dev.yml | 19 ------------------- .github/workflows/main-docker.yml | 10 ---------- bin/build-docker.sh | 3 --- 3 files changed, 32 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 8d50a4640..f5e30323d 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -44,14 +44,6 @@ jobs: - test_dev steps: - uses: actions/checkout@v4 - - name: Set up node & dependencies - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: "npm" - - run: npm ci - - name: Run the TypeScript build - run: npx tsc - uses: docker/setup-buildx-action@v3 - uses: docker/build-push-action@v6 with: @@ -80,17 +72,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Set up node & dependencies - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: "npm" - - - run: npm ci - - - name: Run the TypeScript build - run: npx tsc - - name: Build and export to Docker uses: docker/build-push-action@v6 with: diff --git a/.github/workflows/main-docker.yml b/.github/workflows/main-docker.yml index d8331184f..085ae836e 100644 --- a/.github/workflows/main-docker.yml +++ b/.github/workflows/main-docker.yml @@ -151,16 +151,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - - name: Set up node & dependencies - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: "npm" - - run: npm ci - - name: Run the TypeScript build - run: npx tsc - - name: Login to GHCR uses: docker/login-action@v3 with: diff --git a/bin/build-docker.sh b/bin/build-docker.sh index 892afcd92..d95c289d4 100755 --- a/bin/build-docker.sh +++ b/bin/build-docker.sh @@ -5,9 +5,6 @@ set -e # Fail on any command error VERSION=`jq -r ".version" package.json` SERIES=${VERSION:0:4}-latest -echo "Compiling typescript..." -npx tsc - sudo docker build -t triliumnext/notes:$VERSION --network host -t triliumnext/notes:$SERIES . if [[ $VERSION != *"beta"* ]]; then From 38690053a8cb2c1e4c77d9dae2ef78b5360eccba Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Wed, 5 Mar 2025 08:54:42 +0100 Subject: [PATCH 31/33] build(Docker): improve image size, by deleting unnecessary node_modules fodler from dist folder added a TODO as well, to get rid of this strange step here at some point --- Dockerfile | 4 ++++ Dockerfile.alpine | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/Dockerfile b/Dockerfile index 54db47144..52a087202 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ COPY . . RUN npm ci && \ npm run build:prepare-dist && \ npm cache clean --force && \ + rm -rf dist/node_modules && \ mv dist/* \ start-docker.sh \ package-lock.json \ @@ -17,6 +18,9 @@ RUN npm ci && \ rm -rf /usr/src/app/build #TODO: move package-lock copying into copy-dist +#TODO: improve node_modules handling in copy-dist/Dockerfile -> remove duplicated work +# currently copy-dist will copy certain node_module folders, but in the Dockerfile we delete them again (to keep image size down), +# as we install necessary dependencies in runtime buildstage anyways # Runtime stage FROM node:22.14.0-bullseye-slim diff --git a/Dockerfile.alpine b/Dockerfile.alpine index 43eac8ca7..3d2198bf0 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -10,6 +10,7 @@ COPY . . RUN npm ci && \ npm run build:prepare-dist && \ npm cache clean --force && \ + rm -rf dist/node_modules && \ mv dist/* \ start-docker.sh \ package-lock.json \ @@ -17,6 +18,9 @@ RUN npm ci && \ rm -rf /usr/src/app/build #TODO: move package-lock copying into copy-dist +#TODO: improve node_modules handling in copy-dist/Dockerfile -> remove duplicated work +# currently copy-dist will copy certain node_module folders, but in the Dockerfile we delete them again (to keep image size down), +# as we install necessary dependencies in runtime buildstage anyways # Runtime stage FROM node:22.14.0-alpine From 907b8c503e3062a01c9c6cfb2dfb558a1b3039cb Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Wed, 5 Mar 2025 09:01:21 +0100 Subject: [PATCH 32/33] build: copy package-lock.json into dist folder as well --- Dockerfile | 2 -- Dockerfile.alpine | 2 -- bin/copy-dist.ts | 2 ++ 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 52a087202..c21d9f4f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,11 +13,9 @@ RUN npm ci && \ rm -rf dist/node_modules && \ mv dist/* \ start-docker.sh \ - package-lock.json \ /usr/src/app/ && \ rm -rf /usr/src/app/build -#TODO: move package-lock copying into copy-dist #TODO: improve node_modules handling in copy-dist/Dockerfile -> remove duplicated work # currently copy-dist will copy certain node_module folders, but in the Dockerfile we delete them again (to keep image size down), # as we install necessary dependencies in runtime buildstage anyways diff --git a/Dockerfile.alpine b/Dockerfile.alpine index 3d2198bf0..09efe2529 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -13,11 +13,9 @@ RUN npm ci && \ rm -rf dist/node_modules && \ mv dist/* \ start-docker.sh \ - package-lock.json \ /usr/src/app/ && \ rm -rf /usr/src/app/build -#TODO: move package-lock copying into copy-dist #TODO: improve node_modules handling in copy-dist/Dockerfile -> remove duplicated work # currently copy-dist will copy certain node_module folders, but in the Dockerfile we delete them again (to keep image size down), # as we install necessary dependencies in runtime buildstage anyways diff --git a/bin/copy-dist.ts b/bin/copy-dist.ts index c8911022a..289334321 100644 --- a/bin/copy-dist.ts +++ b/bin/copy-dist.ts @@ -26,6 +26,8 @@ try { "./translations", "./db", "./config-sample.ini", + "./package-lock.json", + "./package.json", "./src/views/", "./src/etapi/etapi.openapi.yaml", "./src/routes/api/openapi.json", From 70e227f4c3b7facc1744d2a8dec87c1b7867e494 Mon Sep 17 00:00:00 2001 From: Panagiotis Papadopoulos Date: Wed, 5 Mar 2025 09:09:54 +0100 Subject: [PATCH 33/33] build(Docker): improve image size, by deleting node-compile-cache --- Dockerfile | 12 ++++++++---- Dockerfile.alpine | 7 +++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index c21d9f4f6..2436a8124 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,9 @@ RUN npm ci && \ mv dist/* \ start-docker.sh \ /usr/src/app/ && \ - rm -rf /usr/src/app/build + rm -rf \ + /usr/src/app/build \ + /tmp/node-compile-cache #TODO: improve node_modules handling in copy-dist/Dockerfile -> remove duplicated work # currently copy-dist will copy certain node_module folders, but in the Dockerfile we delete them again (to keep image size down), @@ -29,14 +31,16 @@ WORKDIR /usr/src/app RUN apt-get update && \ apt-get install -y --no-install-recommends \ gosu && \ - rm -rf /var/lib/apt/lists/* && \ - rm -rf /var/cache/apt/* + rm -rf \ + /var/lib/apt/lists/* \ + /var/cache/apt/* COPY --from=builder /usr/src/app ./ RUN sed -i "/electron/d" package.json && \ npm ci --omit=dev && \ - npm cache clean --force + npm cache clean --force && \ + rm -rf /tmp/node-compile-cache # Configure container EXPOSE 8080 diff --git a/Dockerfile.alpine b/Dockerfile.alpine index 09efe2529..9370bd6da 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -14,7 +14,9 @@ RUN npm ci && \ mv dist/* \ start-docker.sh \ /usr/src/app/ && \ - rm -rf /usr/src/app/build + rm -rf \ + /usr/src/app/build \ + /tmp/node-compile-cache #TODO: improve node_modules handling in copy-dist/Dockerfile -> remove duplicated work # currently copy-dist will copy certain node_module folders, but in the Dockerfile we delete them again (to keep image size down), @@ -32,7 +34,8 @@ COPY --from=builder /usr/src/app ./ RUN sed -i "/electron/d" package.json && \ npm ci --omit=dev && \ - npm cache clean --force + npm cache clean --force && \ + rm -rf /tmp/node-compile-cache # Add application user RUN adduser -s /bin/false node; exit 0