Merge pull request #1225 from TriliumNext/build_improve-ts-webpack

build: improvements for TS & Webpack & Docker

.dockerignore

@@ -1,10 +1,37 @@
-.git
-.idea
+# ignored Files
+.dockerignore
+.editorconfig
+.git*
+.prettier*
+electron*
+entitlements.plist
+forge.config.cjs
+nodemon.json
+renovate.json
+trilium.iml
+Dockerfile
+Dockerfile.*
+npm-debug.log
+/src/**/*.spec.ts
+
+# ignored folders
+/.cache
+/.git
+/.github
+/.idea
+/.vscode
 /bin
+/build
 /dist
 /docs
-/npm-debug.log
-node_modules
+/dump-db
+/e2e
+/integration-tests
+/spec
+/test
+/test-etapi
+/node_modules
 
-src/**/*.ts
-!src/services/asset_path.ts
+
+# exceptions
+!/bin/copy-dist.ts

.github/workflows/dev.yml

@@ -44,16 +44,6 @@ jobs:
       - test_dev
     steps:
       - uses: actions/checkout@v4
-      - name: Set up node & dependencies
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-          cache: "npm"
-      - run: npm ci
-      - name: Run the TypeScript build
-        run: npx tsc
-      - name: Create server-package.json
-        run: cat package.json | grep -v electron > server-package.json
       - uses: docker/setup-buildx-action@v3
       - uses: docker/build-push-action@v6
         with:
@@ -82,20 +72,6 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Set up node & dependencies
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-          cache: "npm"
-
-      - run: npm ci
-
-      - name: Run the TypeScript build
-        run: npx tsc
-
-      - name: Create server-package.json
-        run: cat package.json | grep -v electron > server-package.json
-
       - name: Build and export to Docker
         uses: docker/build-push-action@v6
         with:

.github/workflows/main-docker.yml

@@ -57,9 +57,6 @@ jobs:
       - name: Run the TypeScript build
         run: npx tsc
 
-      - name: Create server-package.json
-        run: cat package.json | grep -v electron > server-package.json
-
       - name: Build and export to Docker
         uses: docker/build-push-action@v6
         with:
@@ -154,18 +151,6 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-
-      - name: Set up node & dependencies
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-          cache: "npm"
-      - run: npm ci
-      - name: Run the TypeScript build
-        run: npx tsc
-      - name: Create server-package.json
-        run: cat package.json | grep -v electron > server-package.json
-
       - name: Login to GHCR
         uses: docker/login-action@v3
         with:

Dockerfile

@@ -1,62 +1,46 @@
 # Build stage
 FROM node:22.14.0-bullseye-slim AS builder
 
-# Configure build dependencies in a single layer
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    autoconf \
-    automake \
-    g++ \
-    gcc \
-    libtool \
-    make \
-    nasm \
-    libpng-dev \
-    python3 \
-    && rm -rf /var/lib/apt/lists/*
-
-WORKDIR /usr/src/app
+WORKDIR /usr/src/app/build
 
 # Copy only necessary files for build
 COPY . .
-COPY server-package.json package.json
 
 # Build and cleanup in a single layer
-RUN cp -R build/src/* src/. && \
-    cp build/docker_healthcheck.js . && \    
-    rm docker_healthcheck.ts && \
-    npm install && \
-    npm run build:webpack && \
-    npm prune --omit=dev && \
+RUN npm ci && \
+    npm run build:prepare-dist && \
     npm cache clean --force && \
-    cp -r src/public/app/doc_notes src/public/app-dist/. && \
-    rm -rf src/public/app/* && \
-    mkdir -p src/public/app/services && \
-    cp -r build/src/public/app/services/mime_type_definitions.js src/public/app/services/mime_type_definitions.js && \
-    rm src/services/asset_path.ts && \
-    rm -r build
+    rm -rf dist/node_modules && \
+    mv dist/* \
+      start-docker.sh \
+      /usr/src/app/ && \
+    rm -rf \
+      /usr/src/app/build \
+      /tmp/node-compile-cache
+
+#TODO: improve node_modules handling in copy-dist/Dockerfile -> remove duplicated work
+#      currently copy-dist will copy certain node_module folders, but in the Dockerfile we delete them again (to keep image size down),
+#      as we install necessary dependencies in runtime buildstage anyways
 
 # Runtime stage
 FROM node:22.14.0-bullseye-slim
 
-# Install only runtime dependencies
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    gosu \
-    && rm -rf /var/lib/apt/lists/* && \
-    rm -rf /var/cache/apt/*
-
 WORKDIR /usr/src/app
 
-# Copy only necessary files from builder
-COPY --from=builder /usr/src/app/node_modules ./node_modules
-COPY --from=builder /usr/src/app/src ./src
-COPY --from=builder /usr/src/app/db ./db
-COPY --from=builder /usr/src/app/docker_healthcheck.js .
-COPY --from=builder /usr/src/app/start-docker.sh .
-COPY --from=builder /usr/src/app/package.json .
-COPY --from=builder /usr/src/app/config-sample.ini .
-COPY --from=builder /usr/src/app/images ./images
-COPY --from=builder /usr/src/app/translations ./translations
-COPY --from=builder /usr/src/app/libraries ./libraries
+# Install only runtime dependencies
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+      gosu && \
+    rm -rf \
+      /var/lib/apt/lists/* \
+      /var/cache/apt/*
+
+COPY --from=builder /usr/src/app ./
+
+RUN sed -i "/electron/d" package.json && \
+    npm ci --omit=dev && \
+    npm cache clean --force && \
+    rm -rf /tmp/node-compile-cache
 
 # Configure container
 EXPOSE 8080

Dockerfile.alpine

@@ -1,38 +1,26 @@
 # Build stage
 FROM node:22.14.0-alpine AS builder
 
-# Configure build dependencies
-RUN apk add --no-cache --virtual .build-dependencies \
-    autoconf \
-    automake \
-    g++ \
-    gcc \
-    libtool \
-    make \
-    nasm \
-    libpng-dev \
-    python3
-
-WORKDIR /usr/src/app
+WORKDIR /usr/src/app/build
 
 # Copy only necessary files for build
 COPY . .
-COPY server-package.json package.json
 
 # Build and cleanup in a single layer
-RUN cp -R build/src/* src/. && \
-    cp build/docker_healthcheck.js . && \
-    rm docker_healthcheck.ts && \
-    npm install && \
-    npm run build:webpack && \
-    npm prune --omit=dev && \
+RUN npm ci && \
+    npm run build:prepare-dist && \
     npm cache clean --force && \
-    cp -r src/public/app/doc_notes src/public/app-dist/. && \
-    rm -rf src/public/app && \
-    mkdir -p src/public/app/services && \
-    cp -r build/src/public/app/services/mime_type_definitions.js src/public/app/services/mime_type_definitions.js && \
-    rm src/services/asset_path.ts && \
-    rm -r build
+    rm -rf dist/node_modules && \
+    mv dist/* \
+      start-docker.sh \
+      /usr/src/app/ && \
+    rm -rf \
+      /usr/src/app/build \
+      /tmp/node-compile-cache
+
+#TODO: improve node_modules handling in copy-dist/Dockerfile -> remove duplicated work
+#      currently copy-dist will copy certain node_module folders, but in the Dockerfile we delete them again (to keep image size down),
+#      as we install necessary dependencies in runtime buildstage anyways
 
 # Runtime stage
 FROM node:22.14.0-alpine
@@ -42,17 +30,12 @@ RUN apk add --no-cache su-exec shadow
 
 WORKDIR /usr/src/app
 
-# Copy only necessary files from builder
-COPY --from=builder /usr/src/app/node_modules ./node_modules
-COPY --from=builder /usr/src/app/src ./src
-COPY --from=builder /usr/src/app/db ./db
-COPY --from=builder /usr/src/app/docker_healthcheck.js .
-COPY --from=builder /usr/src/app/start-docker.sh .
-COPY --from=builder /usr/src/app/package.json .
-COPY --from=builder /usr/src/app/config-sample.ini .
-COPY --from=builder /usr/src/app/images ./images
-COPY --from=builder /usr/src/app/translations ./translations
-COPY --from=builder /usr/src/app/libraries ./libraries
+COPY --from=builder /usr/src/app ./
+
+RUN sed -i "/electron/d" package.json && \
+    npm ci --omit=dev && \
+    npm cache clean --force && \
+    rm -rf /tmp/node-compile-cache
 
 # Add application user
 RUN adduser -s /bin/false node; exit 0

bin/build-docker.sh

@@ -5,11 +5,6 @@ set -e  # Fail on any command error
 VERSION=`jq -r ".version" package.json`
 SERIES=${VERSION:0:4}-latest
 
-cat package.json | grep -v electron > server-package.json
-
-echo "Compiling typescript..."
-npx tsc
-
 sudo docker build -t triliumnext/notes:$VERSION --network host -t triliumnext/notes:$SERIES .
 
 if [[ $VERSION != *"beta"* ]]; then

bin/copy-dist.ts

@@ -2,8 +2,6 @@ import fs from "fs-extra";
 import path from "path";
 
 const DEST_DIR = "./dist";
-const DEST_DIR_SRC = path.join(DEST_DIR, "src");
-const DEST_DIR_NODE_MODULES = path.join(DEST_DIR, "node_modules");
 
 const VERBOSE = process.env.VERBOSE;
 
@@ -13,43 +11,37 @@ function log(...args: any[]) {
     }
 }
 
-async function copyNodeModuleFileOrFolder(source: string) {
-    const adjustedSource = source.substring(13);
-    const destination = path.join(DEST_DIR_NODE_MODULES, adjustedSource);
-
+function copyNodeModuleFileOrFolder(source: string) {
+    const destination = path.join(DEST_DIR, source);
     log(`Copying ${source} to ${destination}`);
-    await fs.ensureDir(path.dirname(destination));
-    await fs.copy(source, destination);
+    fs.ensureDirSync(path.dirname(destination));
+    fs.copySync(source, destination);
 }
 
-const copy = async () => {
-    for (const srcFile of fs.readdirSync("build")) {
-        const destFile = path.join(DEST_DIR, path.basename(srcFile));
-        log(`Copying source ${srcFile} -> ${destFile}.`);
-        fs.copySync(path.join("build", srcFile), destFile, { recursive: true });
-    }
+try {
 
-    const filesToCopy = [
-        "config-sample.ini",
-        "tsconfig.webpack.json",
+    const assetsToCopy = new Set([
+        "./images",
+        "./libraries",
+        "./translations",
+        "./db",
+        "./config-sample.ini",
+        "./package-lock.json",
+        "./package.json",
+        "./src/views/",
         "./src/etapi/etapi.openapi.yaml",
-        "./src/routes/api/openapi.json"
-    ];
-    for (const file of filesToCopy) {
-        log(`Copying ${file}`);
-        await fs.copy(file, path.join(DEST_DIR, file));
-    }
+        "./src/routes/api/openapi.json",
+        "./src/public/icon.png",
+        "./src/public/manifest.webmanifest",
+        "./src/public/robots.txt",
+        "./src/public/fonts",
+        "./src/public/stylesheets",
+        "./src/public/translations"
+    ]);
 
-    const dirsToCopy = ["images", "libraries", "translations", "db"];
-    for (const dir of dirsToCopy) {
-        log(`Copying ${dir}`);
-        await fs.copy(dir, path.join(DEST_DIR, dir));
-    }
-
-    const srcDirsToCopy = ["./src/public", "./src/views", "./build"];
-    for (const dir of srcDirsToCopy) {
-        log(`Copying ${dir}`);
-        await fs.copy(dir, path.join(DEST_DIR_SRC, path.basename(dir)));
+    for (const asset of assetsToCopy) {
+        log(`Copying ${asset}`);
+        fs.copySync(asset, path.join(DEST_DIR, asset));
     }
 
     /**
@@ -58,10 +50,10 @@ const copy = async () => {
     const publicDirsToCopy = ["./src/public/app/doc_notes"];
     const PUBLIC_DIR = path.join(DEST_DIR, "src", "public", "app-dist");
     for (const dir of publicDirsToCopy) {
-        await fs.copy(dir, path.join(PUBLIC_DIR, path.basename(dir)));
+        fs.copySync(dir, path.join(PUBLIC_DIR, path.basename(dir)));
     }
 
-    const nodeModulesFile = [
+    const nodeModulesFile = new Set([
         "node_modules/react/umd/react.production.min.js",
         "node_modules/react/umd/react.development.js",
         "node_modules/react-dom/umd/react-dom.production.min.js",
@@ -71,13 +63,9 @@ const copy = async () => {
         "node_modules/katex/dist/contrib/auto-render.min.js",
         "node_modules/@highlightjs/cdn-assets/highlight.min.js",
         "node_modules/@mind-elixir/node-menu/dist/node-menu.umd.cjs"
-    ];
+    ]);
 
-    for (const file of nodeModulesFile) {
-        await copyNodeModuleFileOrFolder(file);
-    }
-
-    const nodeModulesFolder = [
+    const nodeModulesFolder = new Set([
         "node_modules/@excalidraw/excalidraw/dist/",
         "node_modules/katex/dist/",
         "node_modules/dayjs/",
@@ -104,13 +92,15 @@ const copy = async () => {
         "node_modules/@highlightjs/cdn-assets/languages",
         "node_modules/@highlightjs/cdn-assets/styles",
         "node_modules/leaflet/dist"
-    ];
+    ]);
 
-    for (const folder of nodeModulesFolder) {
-        await copyNodeModuleFileOrFolder(folder);
+
+
+    for (const nodeModuleItem of [...nodeModulesFile, ...nodeModulesFolder]) {
+        copyNodeModuleFileOrFolder(nodeModuleItem);
     }
-};
+    console.log("Copying complete!")
 
-copy()
-    .then(() => console.log("Copying complete!"))
-    .catch((err) => console.error("Error during copy:", err));
+} catch(err) {
+    console.error("Error during copy:", err)
+}

bin/copy-trilium.sh

@@ -14,7 +14,7 @@ fi
 
 # Trigger the TypeScript build
 echo TypeScript build start
-npx tsc
+npm run build:ts
 echo TypeScript build finished
 
 # Copy the TypeScript artifacts

package.json

@@ -43,8 +43,10 @@
     "docs:build-backend": "rimraf ./docs/backend_api && typedoc ./docs/backend_api src/becca/entities/*.ts src/services/backend_script_api.ts src/services/sql.ts",
     "docs:build-frontend": "rimraf ./docs/frontend_api && jsdoc -c jsdoc-conf.json -d ./docs/frontend_api src/public/app/entities/*.js src/public/app/services/frontend_script_api.js src/public/app/widgets/basic_widget.js src/public/app/widgets/note_context_aware_widget.js src/public/app/widgets/right_panel_widget.js",
     "docs:build": "npm run docs:build-backend && npm run docs:build-frontend",
-    "build:webpack": "tsx node_modules/webpack/bin/webpack.js -c webpack.config.ts",
-    "build:prepare-dist": "npm run build:webpack && rimraf ./dist && tsc && tsx ./bin/copy-dist.ts",
+    "build:webpack": "tsx node_modules/webpack/bin/webpack.js -c webpack.config.ts --progress",
+    "build:ts": "tsc -p tsconfig.build.json",
+    "build:clean": "rimraf ./dist ./build",
+    "build:prepare-dist": "npm run build:clean && npm run build:ts && npm run build:webpack && tsx ./bin/copy-dist.ts",
     "test": "npm run client:test && npm run server:test",
     "server:test": "cross-env TRILIUM_ENV=dev TRILIUM_DATA_DIR=./integration-tests/db TRILIUM_INTEGRATION_TEST=memory vitest",
     "server:coverage": "cross-env TRILIUM_ENV=dev TRILIUM_DATA_DIR=./integration-tests/db TRILIUM_INTEGRATION_TEST=memory vitest --coverage",

tsconfig.build.json

@@ -0,0 +1,24 @@
+{
+  "compilerOptions": {
+    "module": "NodeNext",
+    "declaration": false,
+    "sourceMap": true,
+    "outDir": "./dist",
+    "strict": true,
+    "noImplicitAny": true,
+    "resolveJsonModule": true,
+    "allowJs": true,
+    "lib": ["ES2023"],
+    "downlevelIteration": true,
+    "skipLibCheck": true,
+    "esModuleInterop": true,
+    "verbatimModuleSyntax": true
+  },
+  "include": ["./src/**/*.ts", "./src/**/*.js", "./*.ts"],
+  "exclude": [
+	  "./**/*.spec.ts",
+	  "./src/public/**/*",
+    "./*.config.ts",
+  ],
+  "files": ["src/types.d.ts"]
+}

tsconfig.json

@@ -3,7 +3,7 @@
     "module": "NodeNext",
     "declaration": false,
     "sourceMap": true,
-    "outDir": "./build",
+    "outDir": "./dist",
     "strict": true,
     "noImplicitAny": true,
     "resolveJsonModule": true,

tsconfig.webpack.json

@@ -3,7 +3,7 @@
     "module": "NodeNext",
     "declaration": false,
     "sourceMap": true,
-    "outDir": "./build",
+    "outDir": "./dist",
     "strict": true,
     "noImplicitAny": true,
     "resolveJsonModule": true,

webpack.config.ts

@@ -19,7 +19,7 @@ const config: Configuration = {
     },
     output: {
         publicPath: `${assetPath}/app-dist/`,
-        path: path.resolve(rootDir, "src/public/app-dist"),
+        path: path.resolve(rootDir, "dist/src/public/app-dist"),
         filename: "[name].js"
     },
     plugins: [
@@ -75,7 +75,7 @@ const config: Configuration = {
             ".mjs": [".mjs", ".mts"]
         }
     },
-    devtool: "source-map",
+    devtool: "nosources-source-map",
     target: "electron-renderer"
 };