Merge pull request #1436 from TriliumNext/build_copy-dist-trilium-merge

build: port copy-trilium.sh cleanup functionality to cross-platform TS

.dockerignore

@@ -34,10 +34,11 @@ npm-debug.log
 
 # exceptions
 !/bin/copy-dist.ts
-!/bin/electron-forge/sign-windows.cjs
+!/bin/cleanupNodeModules.ts
 
 # temporary exception to make copy-dist inside Docker build not fail
-# TriliumNextTODO: make copy-dist *not* requiring to copy this file for builds other than electron-forge
+# TriliumNextTODO: make copy-dist *not* requiring to copy these file for builds other than electron-forge
 !forge.config.cjs
 !/bin/tpl
-!/bin/electron-forge/desktop.ejs
+!/bin/electron-forge/desktop.ejs
+!/bin/electron-forge/sign-windows.cjs

Dockerfile

@@ -39,8 +39,11 @@ COPY --from=builder /usr/src/app ./
 
 RUN sed -i "/electron/d" package.json && \
     npm ci --omit=dev && \
+    node --experimental-strip-types ./bin/cleanupNodeModules.ts . --skip-prune-dev-deps && \
     npm cache clean --force && \
-    rm -rf /tmp/node-compile-cache
+    rm -rf \
+      /tmp/node-compile-cache \
+      /usr/src/app/bin/cleanupNodeModules.ts
 
 # Configure container
 EXPOSE 8080

Dockerfile.alpine

@@ -34,8 +34,11 @@ COPY --from=builder /usr/src/app ./
 
 RUN sed -i "/electron/d" package.json && \
     npm ci --omit=dev && \
+    node --experimental-strip-types ./bin/cleanupNodeModules.ts . --skip-prune-dev-deps && \
     npm cache clean --force && \
-    rm -rf /tmp/node-compile-cache
+    rm -rf \
+      /tmp/node-compile-cache \
+      /usr/src/app/bin/cleanupNodeModules.ts
 
 # Add application user
 RUN adduser -s /bin/false node; exit 0

bin/build-server.sh

@@ -25,8 +25,16 @@ NODE_VERSION=22.14.0
 
 BUILD_DIR="./build"
 DIST_DIR="./dist"
+CLEANUP_SCRIPT="./bin/cleanupNodeModules.ts"
 
-./bin/copy-trilium.sh
+
+# Trigger the build
+echo "Build start"
+npm run build:prepare-dist
+echo "Build finished"
+
+# pruning of unnecessary files and devDeps in node_modules
+node --experimental-strip-types $CLEANUP_SCRIPT $BUILD_DIR
 
 NODE_FILENAME=node-v${NODE_VERSION}-linux-${ARCH}
 

bin/cleanupNodeModules.ts

@@ -0,0 +1,109 @@
+import fs from "fs-extra";
+import path from "path";
+import type { Dirent } from "fs-extra";
+import { execSync } from "node:child_process";
+
+/**
+ * Example usage with node >= v22:
+ *    node --experimental-strip-types bin/cleanupNodeModules.ts /path/to/build/folder [--skip-prune-dev-deps]
+ * Example usage with tsx:
+ *    tsx bin/cleanupNodeModules.ts /path/to/build/folder [--skip-prune-dev-deps]
+ */
+function main() {
+
+    if (process.argv.length > 4 || process.argv.length < 3) {
+        console.error("Usage: cleanupNodeModules.ts [path-to-build-folder] [--skip-prune-dev-deps]");
+        process.exit(1);
+    }
+
+    const basePath = process.argv[2];
+    const pruneDevDeps = process.argv[3] !== "--skip-prune-dev-deps";
+
+    if (!fs.existsSync(basePath)) {
+        console.error(`Supplied path '${basePath}' does not exist. Aborting.`);
+        process.exit(1);
+    }
+
+    console.log(`Starting pruning of node_modules ${!pruneDevDeps ? '(skipping npm pruning)' : ''} in '${basePath}'...`);
+    cleanupNodeModules(basePath, pruneDevDeps);
+    console.log("Successfully pruned node_modules.");
+}
+
+function cleanupNodeModules(basePath: string, pruneDevDeps: boolean = true) {
+
+    // This needs to run for the server and Docker build,
+    // but needs to be skipped for electron-forge: its
+    // built-in pruning takes care of it already
+    if (pruneDevDeps) {
+        execSync(`npm ci --omit=dev --prefix ${basePath}`);
+    }
+
+    const nodeModulesDirPath = path.join(basePath, "node_modules");
+    const nodeModulesContent = fs.readdirSync(nodeModulesDirPath, { recursive: true, withFileTypes: true });
+    //const libDir = fs.readdirSync(path.join(basePath, "./libraries"), { recursive: true, withFileTypes: true });
+
+    /**
+     * Delete unnecessary folders
+     */
+    const filterableDirs = new Set([
+        "demo",
+        "demos",
+        "doc",
+        "docs",
+        "example",
+        "examples",
+        "test",
+        "tests"
+    ]);
+
+    nodeModulesContent
+        .filter(el => el.isDirectory() && filterableDirs.has(el.name))
+        .forEach(dir => removeDirent(dir));
+
+    /**
+     * Delete unnecessary files based on file extension
+     * TODO filter out useless (README).md files
+     */
+    const filterableFileExt = new Set([
+        "ts",
+        "map"
+    ]);
+
+    nodeModulesContent
+        // TriliumNextTODO: check if we can improve this naive file ext matching, without introducing any additional dependency
+        .filter(el => el.isFile() && filterableFileExt.has(el.name.split(".").at(-1) || ""))
+        .forEach(dir => removeDirent(dir));
+
+
+    /**
+     * Delete specific unnecessary folders
+     * TODO: check if we want removeSync to throw an error, if path does not exist anymore -> currently it will silently fail
+     */
+    const extraFoldersDelete = new Set([
+        path.join(nodeModulesDirPath, ".bin"),
+        path.join(nodeModulesDirPath, "@excalidraw", "excalidraw", "dist", "dev"),
+        path.join(nodeModulesDirPath, "boxicons", "svg"),
+        path.join(nodeModulesDirPath, "boxicons", "node_modules"),
+        path.join(nodeModulesDirPath, "boxicons", "src"),
+        path.join(nodeModulesDirPath, "boxicons", "iconjar"),
+        path.join(nodeModulesDirPath, "@jimp", "plugin-print", "fonts"),
+        path.join(nodeModulesDirPath, "jimp", "dist", "browser") // missing "@" in front of jimp is not a typo here
+    ]);
+
+    nodeModulesContent
+        .filter(el => el.isDirectory() && extraFoldersDelete.has(path.join(el.parentPath, el.name)))
+        .forEach(dir => removeDirent(dir))
+}
+
+
+function removeDirent(el: Dirent) {
+    const elementToDelete = path.join(el.parentPath, el.name);
+    fs.removeSync(elementToDelete);
+
+    if (process.env.VERBOSE) {
+        console.log(`Deleted ${elementToDelete}`);
+    }
+
+}
+
+main()

bin/copy-dist.ts

@@ -11,16 +11,10 @@ function log(...args: any[]) {
     }
 }
 
-function copyNodeModuleFileOrFolder(source: string) {
-    const destination = path.join(DEST_DIR, source);
-    log(`Copying ${source} to ${destination}`);
-    fs.ensureDirSync(path.dirname(destination));
-    fs.copySync(source, destination);
-}
-
 try {
 
     const assetsToCopy = new Set([
+        // copy node_module, to avoid downloading packages a 2nd time during pruning
         "./node_modules",
         "./images",
         "./libraries",
@@ -33,6 +27,7 @@ try {
         "./README.md",
         "./forge.config.cjs",
         "./bin/tpl/",
+        "./bin/cleanupNodeModules.ts",
         "./bin/electron-forge/desktop.ejs",
         "./bin/electron-forge/sign-windows.cjs",
         "./src/views/",
@@ -61,50 +56,10 @@ try {
         fs.copySync(dir, path.join(PUBLIC_DIR, path.basename(dir)));
     }
 
-    const nodeModulesFile = new Set([
-        "node_modules/react/umd/react.production.min.js",
-        "node_modules/react/umd/react.development.js",
-        "node_modules/react-dom/umd/react-dom.production.min.js",
-        "node_modules/react-dom/umd/react-dom.development.js",
-        "node_modules/katex/dist/katex.min.js",
-        "node_modules/katex/dist/contrib/mhchem.min.js",
-        "node_modules/katex/dist/contrib/auto-render.min.js",
-        "node_modules/@highlightjs/cdn-assets/highlight.min.js",
-    ]);
-
-    const nodeModulesFolder = new Set([
-        "node_modules/@excalidraw/excalidraw/dist/prod/fonts/",
-        "node_modules/katex/dist/",
-        "node_modules/dayjs/",
-        "node_modules/boxicons/css/",
-        "node_modules/boxicons/fonts/",
-        "node_modules/jquery/dist/",
-        "node_modules/jquery-hotkeys/",
-        "node_modules/split.js/dist/",
-        "node_modules/i18next/",
-        "node_modules/i18next-http-backend/",
-        "node_modules/vanilla-js-wheel-zoom/dist/",
-        "node_modules/mark.js/dist/",
-        "node_modules/normalize.css/",
-        "node_modules/jquery.fancytree/dist/",
-        "node_modules/autocomplete.js/dist/",
-        "node_modules/codemirror/lib/",
-        "node_modules/codemirror/addon/",
-        "node_modules/codemirror/mode/",
-        "node_modules/codemirror/keymap/",
-        "node_modules/@highlightjs/cdn-assets/languages",
-        "node_modules/@highlightjs/cdn-assets/styles",
-        "node_modules/leaflet/dist"
-    ]);
-
-
-
-    for (const nodeModuleItem of [...nodeModulesFile, ...nodeModulesFolder]) {
-        copyNodeModuleFileOrFolder(nodeModuleItem);
-    }
     console.log("Copying complete!")
 
 } catch(err) {
     console.error("Error during copy:", err)
     process.exit(1)
 }
+

bin/copy-trilium.sh

@@ -1,44 +0,0 @@
-#!/usr/bin/env bash
-
-set -e  # Fail on any command error
-shopt -s globstar
-
-BUILD_DIR="./build"
-
-if ! [[ $(which npm) ]]; then
-    echo "Missing npm"
-    exit 1
-fi
-
-# Trigger the build
-echo Build start
-npm run build:prepare-dist
-echo Build finished
-
-# Patch package.json main
-sed -i 's|./dist/electron-main.js|electron-main.js|g' "$BUILD_DIR/package.json"
-
-# run in subshell (so we return to original dir)
-(cd $BUILD_DIR && npm ci --omit=dev)
-
-if [[ -d "$BUILD_DIR"/node_modules ]]; then
-    # cleanup of useless files in dependencies
-    for d in 'image-q/demo' \
-        '@excalidraw/excalidraw/dist/excalidraw-assets-dev' '@excalidraw/excalidraw/dist/excalidraw.development.js' '@excalidraw/excalidraw/dist/excalidraw-with-preact.development.js' \
-        'mermaid/dist/mermaid.js' \
-        'boxicons/svg' 'boxicons/node_modules/react'/* \
-        '@jimp/plugin-print/fonts' 'jimp/browser' 'jimp/fonts'; do
-        [[ -e "$BUILD_DIR"/node_modules/"$d" ]] && rm -r "$BUILD_DIR"/node_modules/"$d"
-    done
-
-    # delete all tests (there are often large images as test file for jimp etc.)
-    for d in 'test' 'docs' 'demo' 'example'; do
-        find "$BUILD_DIR"/node_modules -name "$d" -exec rm -rf {} +
-    done
-fi
-
-find $BUILD_DIR/libraries -name "*.map" -type f -delete
-find $BUILD_DIR/node_modules -name "*.map" -type f -delete
-find $BUILD_DIR -name "*.ts" -type f -delete
-
-unset f d BUILD_DIR

forge.config.cjs

@@ -1,5 +1,6 @@
 const path = require("path");
 const fs = require("fs-extra");
+const { execSync } = require("child_process");
 
 const APP_NAME = "TriliumNext Notes";
 const BIN_PATH = path.normalize("./bin/electron-forge");
@@ -39,6 +40,22 @@ module.exports = {
             "translations/",
             "node_modules/@highlightjs/cdn-assets/styles"
         ],
+        afterPrune: [
+            (buildPath, _electronVersion, _platform, _arch, callback) => {
+                // buildPath is a temporary directory that electron-packager creates - it's in the form of
+                // /tmp/electron-packager/tmp-SjJl0s/resources/app
+                try {
+                    const cleanupNodeModulesScript = path.join(buildPath, "bin", "cleanupNodeModules.ts");
+                    // we don't have access to any devDeps like 'tsx' here, so use the built-in '--experimental-strip-types' flag instead
+                    const command = `node --experimental-strip-types ${cleanupNodeModulesScript} "${buildPath}" --skip-prune-dev-deps`;
+                    // execSync throws, if above returns any non-zero exit code
+                    execSync(command);
+                    callback()
+                } catch(err) {
+                    callback(err)
+                }
+            }
+        ],
         afterComplete: [
             (buildPath, _electronVersion, platform, _arch, callback) => {
                 // Only move resources on non-macOS platforms

package.json

@@ -38,10 +38,9 @@
     "electron:switch": "electron-rebuild",
     "docs:edit": "cross-env NODE_OPTIONS=\"--import tsx\" TRILIUM_DATA_DIR=./data-docs TRILIUM_ENV=dev TRILIUM_PORT=37741 electron ./electron-docs-main.ts .",
     "docs:edit-nix": "electron-rebuild --version 33.3.1 && cross-env NODE_OPTIONS=\"--import tsx\" TRILIUM_DATA_DIR=./data-docs TRILIUM_PORT=37741 TRILIUM_ENV=dev nix-shell -p electron_33 --run \"electron ./electron-docs-main.ts .\"",
-    "electron-forge:prepare": "npm run build:prepare-dist",
-    "electron-forge:start": "npm run electron-forge:prepare && cd ./build && electron-forge start",
-    "electron-forge:make": "npm run electron-forge:prepare && cross-env DEBUG=electron-windows-installer:* electron-forge make ./build",
-    "electron-forge:package": "npm run electron-forge:prepare && cd ./build && electron-forge package",
+    "electron-forge:start": "npm run build:prepare-dist && cd ./build && electron-forge start",
+    "electron-forge:make": "npm run build:prepare-dist && cross-env DEBUG=electron-windows-installer:* electron-forge make ./build",
+    "electron-forge:package": "npm run build:prepare-dist && cd ./build && electron-forge package",
     "docs:build-backend": "rimraf ./docs/backend_api && typedoc ./docs/backend_api src/becca/entities/*.ts src/services/backend_script_api.ts src/services/sql.ts",
     "docs:build-frontend": "rimraf ./docs/frontend_api && jsdoc -c jsdoc-conf.json -d ./docs/frontend_api src/public/app/entities/*.js src/public/app/services/frontend_script_api.js src/public/app/widgets/basic_widget.js src/public/app/widgets/note_context_aware_widget.js src/public/app/widgets/right_panel_widget.js",
     "docs:build": "npm run docs:build-backend && npm run docs:build-frontend",
@@ -232,7 +231,6 @@
     "lorem-ipsum": "2.0.8",
     "mind-elixir": "4.4.3",
     "mini-css-extract-plugin": "2.9.2",
-    "node-abi": "4.2.0",
     "nodemon": "3.1.9",
     "postcss-loader": "8.1.1",
     "prettier": "3.5.3",