From cadd78524c2188320c99ed9420e1d5fbe412b63c Mon Sep 17 00:00:00 2001
From: Elian Doran <contact@eliandoran.me>
Date: Sat, 22 Feb 2025 01:01:15 +0200
Subject: [PATCH] feat(import/single): support UTF-16 LE with BOM for code
 notes

---
 .../import/samples/UTF-16LE Code Note.json    | Bin 0 -> 50 bytes
 src/services/import/single.spec.ts            |  27 ++++++++++++------
 src/services/import/single.ts                 |   2 +-
 3 files changed, 20 insertions(+), 9 deletions(-)
 create mode 100644 src/services/import/samples/UTF-16LE Code Note.json

diff --git a/src/services/import/samples/UTF-16LE Code Note.json b/src/services/import/samples/UTF-16LE Code Note.json
new file mode 100644
index 0000000000000000000000000000000000000000..a9d06ee69761ef2fd6f752ba345bc3dd19329a1a
GIT binary patch
literal 50
tcmezWubP32K>-St7(5tK8FGLypFxSi3Mi_?P!41l0Yy@PY%Ydc1_1Zq2#5dx

literal 0
HcmV?d00001

diff --git a/src/services/import/single.spec.ts b/src/services/import/single.spec.ts
index e650c8e98..f4e0cff11 100644
--- a/src/services/import/single.spec.ts
+++ b/src/services/import/single.spec.ts
@@ -10,15 +10,17 @@ import cls from "../cls.js";
 import sql_init from "../sql_init.js";
 import { initializeTranslations } from "../i18n.js";
 import single from "./single.js";
+import stripBom from "strip-bom";
 const scriptDir = dirname(fileURLToPath(import.meta.url));
 
-async function testImport(fileName: string, mimetype: string): Promise<BNote> {
-    const mdxSample = fs.readFileSync(path.join(scriptDir, "samples", fileName));
+async function testImport(fileName: string, mimetype: string) {
+    const buffer = fs.readFileSync(path.join(scriptDir, "samples", fileName));
     const taskContext = TaskContext.getInstance("import-mdx", "import", {
-        textImportedAsText: true
+        textImportedAsText: true,
+        codeImportedAsCode: true
     });
 
-    return new Promise<BNote>((resolve, reject) => {
+    return new Promise<{ buffer: Buffer, importedNote: BNote }>((resolve, reject) => {
         cls.init(async () => {
             const rootNote = becca.getNote("root");
             if (!rootNote) {
@@ -28,9 +30,12 @@ async function testImport(fileName: string, mimetype: string): Promise<BNote> {
             const importedNote = single.importSingleFile(taskContext, {
                 originalname: fileName,
                 mimetype,
-                buffer: mdxSample
+                buffer: buffer
             }, rootNote as BNote);
-            resolve(importedNote);
+            resolve({
+                buffer,
+                importedNote
+            });
         });
     });
 }
@@ -43,16 +48,22 @@ describe("processNoteContent", () => {
     });
 
     it("treats single MDX as Markdown", async () => {
-        const importedNote = await testImport("Text Note.mdx", "text/mdx");
+        const { importedNote } = await testImport("Text Note.mdx", "text/mdx");
         expect(importedNote.mime).toBe("text/html");
         expect(importedNote.type).toBe("text");
         expect(importedNote.title).toBe("Text Note");
     });
 
     it("supports HTML note with UTF-16 (w/ BOM) from Microsoft Outlook", async () => {
-        const importedNote = await testImport("IREN Reports Q2 FY25 Results.htm", "text/html");
+        const { importedNote } = await testImport("IREN Reports Q2 FY25 Results.htm", "text/html");
         expect(importedNote.mime).toBe("text/html");
         expect(importedNote.title).toBe("IREN Reports Q2 FY25 Results");
         expect(importedNote.getContent().toString().substring(0, 5)).toEqual("<html");
     });
+
+    it("supports code note with UTF-16", async () => {
+        const { importedNote, buffer } = await testImport("UTF-16LE Code Note.json", "application/json");
+        expect(importedNote.mime).toBe("application/json");
+        expect(importedNote.getContent().toString()).toStrictEqual(stripBom(buffer.toString("utf-16le")));
+    });
 })
diff --git a/src/services/import/single.ts b/src/services/import/single.ts
index e4ff7a068..f9b96cfcd 100644
--- a/src/services/import/single.ts
+++ b/src/services/import/single.ts
@@ -71,7 +71,7 @@ function importFile(taskContext: TaskContext, file: File, parentNote: BNote) {
 
 function importCodeNote(taskContext: TaskContext, file: File, parentNote: BNote) {
     const title = getNoteTitle(file.originalname, !!taskContext.data?.replaceUnderscoresWithSpaces);
-    const content = file.buffer.toString("utf-8");
+    const content = processStringOrBuffer(file.buffer);
     const detectedMime = mimeService.getMime(file.originalname) || file.mimetype;
     const mime = mimeService.normalizeMimeType(detectedMime);