diff --git a/apps/edit-docs/tsconfig.app.json b/apps/edit-docs/tsconfig.app.json index a232431d0..2479661aa 100644 --- a/apps/edit-docs/tsconfig.app.json +++ b/apps/edit-docs/tsconfig.app.json @@ -18,6 +18,9 @@ "eslint.config.mjs" ], "references": [ + { + "path": "../server/tsconfig.app.json" + }, { "path": "../desktop/tsconfig.app.json" }, diff --git a/apps/edit-docs/tsconfig.json b/apps/edit-docs/tsconfig.json index 56baa3f4e..de6607550 100644 --- a/apps/edit-docs/tsconfig.json +++ b/apps/edit-docs/tsconfig.json @@ -3,6 +3,9 @@ "files": [], "include": [], "references": [ + { + "path": "../server" + }, { "path": "../desktop" }, diff --git a/apps/server/package.json b/apps/server/package.json index 95e09a807..3b246c790 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -53,7 +53,7 @@ "@anthropic-ai/sdk": "0.40.1", "@braintree/sanitize-url": "7.1.1", "@triliumnext/commons": "workspace:*", - "@triliumnext/express-partial-content": "1.0.1", + "@triliumnext/express-partial-content": "workspace:*", "@triliumnext/turndown-plugin-gfm": "workspace:*", "archiver": "7.0.1", "async-mutex": "0.5.0", diff --git a/apps/server/src/services/import/mime.spec.ts b/apps/server/src/services/import/mime.spec.ts index 5b04d0f2a..223517d32 100644 --- a/apps/server/src/services/import/mime.spec.ts +++ b/apps/server/src/services/import/mime.spec.ts @@ -45,6 +45,11 @@ describe("#getMime", () => { ["test.zip"], "application/zip" ], + [ + "MP4 videos are supported", + ["video.mp4"], "video/mp4" + ], + [ "unknown MIME type not recognized by mimeTypes.lookup", ["test.fake"], false diff --git a/apps/server/src/services/import/mime.ts b/apps/server/src/services/import/mime.ts index 842e12c3e..9b9895ea9 100644 --- a/apps/server/src/services/import/mime.ts +++ b/apps/server/src/services/import/mime.ts @@ -71,7 +71,8 @@ const EXTENSION_TO_MIME = new Map([ [".ts", "text/x-typescript"], [".excalidraw", "application/json"], [".mermaid", "text/vnd.mermaid"], - [".mmd", "text/vnd.mermaid"] + [".mmd", "text/vnd.mermaid"], + [".mp4", "video/mp4"] // https://github.com/jshttp/mime-types/issues/138 ]); /** @returns false if MIME is not detected */ diff --git a/apps/server/tsconfig.app.json b/apps/server/tsconfig.app.json index 7f5a65bec..2a567c5f2 100644 --- a/apps/server/tsconfig.app.json +++ b/apps/server/tsconfig.app.json @@ -36,6 +36,9 @@ { "path": "../../packages/turndown-plugin-gfm/tsconfig.lib.json" }, + { + "path": "../../packages/express-partial-content/tsconfig.lib.json" + }, { "path": "../../packages/commons/tsconfig.lib.json" } diff --git a/apps/server/tsconfig.json b/apps/server/tsconfig.json index 4219c7535..baacd3fa5 100644 --- a/apps/server/tsconfig.json +++ b/apps/server/tsconfig.json @@ -6,6 +6,9 @@ { "path": "../../packages/turndown-plugin-gfm" }, + { + "path": "../../packages/express-partial-content" + }, { "path": "../../packages/commons" }, diff --git a/packages/express-partial-content/.swcrc b/packages/express-partial-content/.swcrc new file mode 100644 index 000000000..73fa88b8d --- /dev/null +++ b/packages/express-partial-content/.swcrc @@ -0,0 +1,22 @@ +{ + "jsc": { + "target": "es2017", + "parser": { + "syntax": "typescript", + "decorators": true, + "dynamicImport": true + }, + "transform": { + "decoratorMetadata": true, + "legacyDecorator": true + }, + "keepClassNames": true, + "externalHelpers": true, + "loose": true + }, + "module": { + "type": "commonjs" + }, + "sourceMaps": true, + "exclude": ["jest.config.ts",".*\\.spec.tsx?$",".*\\.test.tsx?$","./src/jest-setup.ts$","./**/jest-setup.ts$",".*.js$"] +} diff --git a/packages/express-partial-content/LICENSE b/packages/express-partial-content/LICENSE new file mode 100644 index 000000000..bd6fa27a5 --- /dev/null +++ b/packages/express-partial-content/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Sukant Gujar + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/express-partial-content/README.md b/packages/express-partial-content/README.md new file mode 100644 index 000000000..8fa64f247 --- /dev/null +++ b/packages/express-partial-content/README.md @@ -0,0 +1,149 @@ +![Travis CI Status](https://travis-ci.com/SukantGujar/express-partial-content.svg?branch=master) + +# About + +A HTTP 206 Partial Content handler to serve any readable stream partially in Express. + +Based on this blog post: https://www.codeproject.com/Articles/813480/HTTP-Partial-Content-In-Node-js. + +# Installation + +`yarn add express-partial-content` + +OR + +`npm install express-partial-content` + +> Note: `Express` package is a peer dependency for `express-partial-content` and must be present in dependencies of the host package. + +# Usage + +From the `express-file-server` example: + +1. Implement a `ContentProvider` function which prepares and returns a `Content` object: + + import { promisify } from "util"; + import fs from "fs"; + import { Range, ContentDoesNotExistError, ContentProvider } from "express-partial-content"; + import {logger} from "./logger"; + + const statAsync = promisify(fs.stat); + const existsAsync = promisify(fs.exists); + + export const fileContentProvider: ContentProvider = async (req: Request) => { + // Read file name from route params. + const fileName = req.params.name; + const file = `${__dirname}/files/${fileName}`; + if (!(await existsAsync(file))) { + throw new ContentDoesNotExistError(`File doesn't exist: ${file}`); + } + const stats = await statAsync(file); + const totalSize = stats.size; + const mimeType = "application/octet-stream"; + const getStream = (range?: Range) => { + if (!range) { + // Request if for complete content. + return fs.createReadStream(file); + } + // Partial content request. + const { start, end } = range; + logger.debug(`start: ${start}, end: ${end}`); + return fs.createReadStream(file, { start, end }); + }; + return { + fileName, + totalSize, + mimeType, + getStream + }; + }; + +2. In your express code, use `createPartialContentHandler` factory method to generate an express handler for serving partial content for the route of your choice: + + import {createPartialContentHandler} from "express-partial-content"; + import {logger} from "./logger"; + + const handler = createPartialContentHandler(fileContentProvider, logger); + + const app = express(); + const port = 8080; + + // File name is a route param. + app.get("/files/:name", handler); + + app.listen(port, () => { + logger.debug("Server started!"); + }); + +3. Run your server and use a multi-part/multi-connection download utility like [aria2c](https://aria2.github.io/) to test it: + + aria -x5 -k1M http://localhost:8080/files/readme.txt + +# Examples + +There one examples in the `src/examples` folder: + +1. `express-file-server`: Implements a file based `ContentProvider`. + +## Running the examples: + +1. `express-file-server`: Run the following commands, the server will listen on http://localhost:8080/. + + yarn build:dev + yarn copy-assets + yarn run:examples:file + +## Connecting to the running server: + +Browse to `https://localhost:8080/files/readme.txt` + +# Reference + +## createPartialContentHandler function: + +This is a factory method which generates a partial content handler for express routes. + +### Arguments: + +- `contentProvider`: An `async` function which returns a Promise resolved to a `Content` object (see below). +- `logger`: Any logging implementation which has a `debug(message:string, extra: any)` method. Either `winston` or `bunyan` loggers should work. + +### Returns: + +- Express Route Handler: `createPartialContentHandler` returns an express handler which can be mapped to an Express route to serve partial content. + +## ContentProvider function: + +This function _needs to be implemented by you_. It's purpose is to fetch and return `Content` object containing necessary metadata and methods to stream the content partially. This method is invoked by the express handler (returned by `createPartialContentHandler`) on each request. + +### Arguments: + +- `Request`: It receives the `Request` object as it's only input. Use the information available in `Request` to find the requested content, e.g. through `Request.params` or query string, headers etc. + +### Returns: + +- `Promise`: See below. + +### Throws: + +- `ContentDoesNotExistError`: Throw this to indicate that the content doesn't exist. The generated express handler will return a 404 in this case. + > Note: Any message provided to the `ContentDoesNotExistError` object is returned to the client. + +## Content object: + +This object contains metadata and methods which describe the content. The `ContentProvider` method builds and returns it. + +### Properties: + +All the properties of this object are used to return content metadata to the client as various `Response` headers. + +- `fileName`: Used as the `Content-Disposition` header's `filename` value. +- `mimeType`: Used as the `Content-Type` header value. +- `totalSize`: Used as the `Content-Length` header value. + +### Methods: + +- `getStream(range?: Range)`: This method should return a readable stream initialized to the provided `range` (optional). You need to handle two cases: + + - range is `null`: When `range` is not-specified, the client is requesting the full content. In this case, return the stream as it is. + - range is `{start, end}`: When client requests partial content, the `start` and `end` values will point to the corresponding byte positions (0 based and inclusive) of the content. You need to return stream limited to these positions. diff --git a/packages/express-partial-content/eslint.config.mjs b/packages/express-partial-content/eslint.config.mjs new file mode 100644 index 000000000..b0d78db72 --- /dev/null +++ b/packages/express-partial-content/eslint.config.mjs @@ -0,0 +1,23 @@ +import baseConfig from "../../eslint.config.mjs"; + +export default [ + ...baseConfig, + { + "files": [ + "**/*.json" + ], + "rules": { + "@nx/dependency-checks": [ + "error", + { + "ignoredFiles": [ + "{projectRoot}/eslint.config.{js,cjs,mjs}" + ] + } + ] + }, + "languageOptions": { + "parser": (await import('jsonc-eslint-parser')) + } + } +]; diff --git a/packages/express-partial-content/package.json b/packages/express-partial-content/package.json new file mode 100644 index 000000000..96c49d63a --- /dev/null +++ b/packages/express-partial-content/package.json @@ -0,0 +1,47 @@ +{ + "name": "@triliumnext/express-partial-content", + "description": "A partial content handler implementation for any readable stream with Express. Based on this blog post: https://www.codeproject.com/Articles/813480/HTTP-Partial-Content-In-Node-js.", + "license": "MIT", + "version": "1.1.0", + "type": "module", + "private": true, + "main": "./dist/index.js", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + "./package.json": "./package.json", + ".": { + "development": "./src/index.ts", + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + } + }, + "keywords": [ + "partial-content", + "206", + "stream", + "typescript" + ], + "nx": { + "name": "express-partial-content", + "targets": { + "build": { + "executor": "@nx/js:swc", + "outputs": [ + "{options.outputPath}" + ], + "options": { + "outputPath": "packages/express-partial-content/dist", + "tsConfig": "packages/express-partial-content/tsconfig.lib.json", + "packageJson": "packages/express-partial-content/package.json", + "main": "packages/express-partial-content/src/index.ts", + "stripLeadingPaths": true + } + } + } + }, + "dependencies": { + "tslib": "^2.3.0" + } +} diff --git a/packages/express-partial-content/src/Content.ts b/packages/express-partial-content/src/Content.ts new file mode 100644 index 000000000..bba8fed24 --- /dev/null +++ b/packages/express-partial-content/src/Content.ts @@ -0,0 +1,22 @@ +import type { Range } from "./Range.js"; +import { Stream } from "stream"; +export interface Content { + /** + * Returns a readable stream based on the provided range (optional). + * @param {Range} range The start-end range of stream data. + * @returns {Stream} A readable stream + */ + getStream(range?: Range): Stream; + /** + * Total size of the content + */ + readonly totalSize: number; + /** + * Mime type to be sent in Content-Type header + */ + readonly mimeType: string; + /** + * File name to be sent in Content-Disposition header + */ + readonly fileName: string; +}; diff --git a/packages/express-partial-content/src/ContentDoesNotExistError.ts b/packages/express-partial-content/src/ContentDoesNotExistError.ts new file mode 100644 index 000000000..ebd8ffdde --- /dev/null +++ b/packages/express-partial-content/src/ContentDoesNotExistError.ts @@ -0,0 +1,2 @@ +export class ContentDoesNotExistError extends Error { +} diff --git a/packages/express-partial-content/src/ContentProvider.ts b/packages/express-partial-content/src/ContentProvider.ts new file mode 100644 index 000000000..6fb2648c3 --- /dev/null +++ b/packages/express-partial-content/src/ContentProvider.ts @@ -0,0 +1,6 @@ +import type { Request } from "express"; +import type { Content } from "./Content.js"; +/** + * @type {function (Request): Promise} + */ +export type ContentProvider = (req: Request) => Promise; diff --git a/packages/express-partial-content/src/Logger.ts b/packages/express-partial-content/src/Logger.ts new file mode 100644 index 000000000..c0f628863 --- /dev/null +++ b/packages/express-partial-content/src/Logger.ts @@ -0,0 +1,3 @@ +export interface Logger { + debug(message: string, extra?: any): void; +} diff --git a/packages/express-partial-content/src/Range.ts b/packages/express-partial-content/src/Range.ts new file mode 100644 index 000000000..f6c11015c --- /dev/null +++ b/packages/express-partial-content/src/Range.ts @@ -0,0 +1,4 @@ +export type Range = { + start: number; + end: number; +}; diff --git a/packages/express-partial-content/src/RangeParserError.ts b/packages/express-partial-content/src/RangeParserError.ts new file mode 100644 index 000000000..842efbc6d --- /dev/null +++ b/packages/express-partial-content/src/RangeParserError.ts @@ -0,0 +1,5 @@ +export class RangeParserError extends Error { + constructor(start: any, end: any) { + super(`Invalid start and end values: ${start}-${end}.`); + } +} diff --git a/packages/express-partial-content/src/createPartialContentHandler.spec.ts b/packages/express-partial-content/src/createPartialContentHandler.spec.ts new file mode 100644 index 000000000..d0c5026b0 --- /dev/null +++ b/packages/express-partial-content/src/createPartialContentHandler.spec.ts @@ -0,0 +1,181 @@ +import * as utils from "./utils.js"; +import * as ParseRangeHeaderExports from "./parseRangeHeader.js"; +import { ContentDoesNotExistError } from "./ContentDoesNotExistError.js"; +import { createPartialContentHandler } from "./createPartialContentHandler.js"; +import type { ContentProvider } from "./ContentProvider.js"; +import type { Logger } from "./Logger.js"; +import type { Request, Response } from "express"; +import type { Content } from "./Content.js"; +import { Stream } from "stream"; +import type { Range } from "./Range.js"; +import type { MockInstance } from "vitest"; + +describe("createPartialContentHandler tests", () => { + let logger: Logger; + beforeEach(() => { + logger = { + debug: vi.fn() as (message: string, extra?: any) => void + }; + }); + afterEach(() => { + vi.restoreAllMocks(); + }); + it("returns a handler", () => { + const contentProvider = vi.fn().mockResolvedValue({}) as ContentProvider; + const handler = createPartialContentHandler(contentProvider, logger); + expect(typeof handler === "function"); + }); + + describe("handler tests", () => { + let req: Request; + let res: Response; + let statusSpy: MockInstance; + let sendSpy: MockInstance; + let sendStatusSpy: MockInstance; + beforeEach(() => { + req = {} as Request; + res = { + status: (code: number) => res, + send: (message: string) => res, + sendStatus: (code: number) => res, + setHeader: vi.fn() as (name: string, value: string) => void + } as Response; + statusSpy = vi.spyOn(res, "status"); + sendSpy = vi.spyOn(res, "send"); + sendStatusSpy = vi.spyOn(res, "sendStatus"); + }); + it("invokes contentProvider with the specified request", async () => { + const contentProvider = vi.fn().mockResolvedValue({}) as ContentProvider; + const handler = createPartialContentHandler(contentProvider, logger); + try { + await handler(req, res); + } catch {} + expect(contentProvider).toHaveBeenCalledExactlyOnceWith(req); + }); + it("returns 404 if contentProvider throws ContentDoesNotExistError error", async () => { + const error = new ContentDoesNotExistError("404-File not found!"); + const contentProvider = vi.fn().mockRejectedValue(error) as ContentProvider; + const handler = createPartialContentHandler(contentProvider, logger); + try { + await handler(req, res); + expect(statusSpy).toHaveBeenCalledExactlyOnceWith(404); + expect(sendSpy).toHaveBeenCalledExactlyOnceWith(error.message); + } catch { + expect(false); + } + }); + it("returns 500 if contentProvider throws any other error", async () => { + const error = new Error("Something went wrong!"); + const contentProvider = vi.fn().mockRejectedValue(error) as ContentProvider; + const handler = createPartialContentHandler(contentProvider, logger); + try { + await handler(req, res); + expect(sendStatusSpy).toHaveBeenCalledExactlyOnceWith(500); + } catch { + expect(false); + } + }); + it("returns 416 if parseRangeHeader throws RangeParserError error", async () => { + const contentProvider = vi.fn().mockResolvedValue({}) as ContentProvider; + const handler = createPartialContentHandler(contentProvider, logger); + req.headers = { range: "bytes=30-10" }; + try { + await handler(req, res); + expect(statusSpy).toHaveBeenCalledExactlyOnceWith(416); + } catch { + expect(false); + } + }); + it("returns 500 if parseRangeHeader throws other errors", async () => { + const parseRangeHeaderStub = vi + .spyOn(ParseRangeHeaderExports, "parseRangeHeader") + .mockImplementation(() => { + throw new Error("Something went wrong!") + }); + const contentProvider = vi.fn().mockResolvedValue({}) as ContentProvider; + const handler = createPartialContentHandler(contentProvider, logger); + try { + await handler(req, res); + expect(sendStatusSpy).toHaveBeenCalledExactlyOnceWith(500); + } catch { + expect(false); + } + }); + it("returns correct response if range is not specified", async () => { + const result = ({ + pipe() { + return result; + } + } as any) as Stream; + const content: Content = { + fileName: "file.txt", + totalSize: 10, + mimeType: "text/plain", + getStream(range?: Range) { + return result; + } + }; + const pipeSpy = vi.spyOn(result, "pipe"); + const getStreamSpy = vi.spyOn(content, "getStream"); + const contentProvider = vi.fn().mockResolvedValue(content) as ContentProvider; + const handler = createPartialContentHandler(contentProvider, logger); + const setContentTypeHeaderSpy = vi.spyOn(utils, "setContentTypeHeader"); + const setContentDispositionHeaderSpy = vi.spyOn(utils, "setContentDispositionHeader"); + const setAcceptRangesHeaderSpy = vi.spyOn(utils, "setAcceptRangesHeader"); + const setContentLengthHeaderSpy = vi.spyOn(utils, "setContentLengthHeader"); + const setContentRangeHeaderSpy = vi.spyOn(utils, "setContentRangeHeader"); + try { + await handler(req, res); + expect(setContentTypeHeaderSpy).toHaveBeenCalledExactlyOnceWith(content.mimeType, res); + expect(setContentDispositionHeaderSpy).toHaveBeenCalledExactlyOnceWith(content.fileName, res); + expect(setAcceptRangesHeaderSpy).toHaveBeenCalledExactlyOnceWith(res); + expect(setContentLengthHeaderSpy).toHaveBeenCalledExactlyOnceWith(content.totalSize, res); + expect(getStreamSpy).toHaveBeenCalledExactlyOnceWith(); + expect(pipeSpy).toHaveBeenCalledExactlyOnceWith(res); + expect(setContentRangeHeaderSpy).not.toHaveBeenCalled(); + } catch { + expect(false); + } + }); + it("returns correct partial response if range is specified", async () => { + req.headers = { + range: "bytes=0-5" + }; + const result = ({ + pipe() { + return result; + } + } as any) as Stream; + const content: Content = { + fileName: "file.txt", + totalSize: 10, + mimeType: "text/plain", + getStream(range?: Range) { + return result; + } + }; + const range = { start: 0, end: 5 }; + const pipeSpy = vi.spyOn(result, "pipe"); + const getStreamSpy = vi.spyOn(content, "getStream"); + const contentProvider = vi.fn().mockResolvedValue(content) as ContentProvider; + const handler = createPartialContentHandler(contentProvider, logger); + const setContentTypeHeaderSpy = vi.spyOn(utils, "setContentTypeHeader"); + const setContentDispositionHeaderSpy = vi.spyOn(utils, "setContentDispositionHeader"); + const setAcceptRangesHeaderSpy = vi.spyOn(utils, "setAcceptRangesHeader"); + const setContentLengthHeaderSpy = vi.spyOn(utils, "setContentLengthHeader"); + const setContentRangeHeaderSpy = vi.spyOn(utils, "setContentRangeHeader"); + try { + await handler(req, res); + expect(setContentTypeHeaderSpy).toHaveBeenCalledExactlyOnceWith(content.mimeType, res); + expect(setContentDispositionHeaderSpy).toHaveBeenCalledExactlyOnceWith(content.fileName, res); + expect(setAcceptRangesHeaderSpy).toHaveBeenCalledExactlyOnceWith(res); + expect(setContentRangeHeaderSpy).toHaveBeenCalledExactlyOnceWith(range, content.totalSize, res); + expect(setContentLengthHeaderSpy).toHaveBeenCalledExactlyOnceWith(6, res); + expect(getStreamSpy).toHaveBeenCalledExactlyOnceWith(range); + expect(pipeSpy).toHaveBeenCalledExactlyOnceWith(res); + } catch { + expect(false); + } + }); + }); +}); diff --git a/packages/express-partial-content/src/createPartialContentHandler.ts b/packages/express-partial-content/src/createPartialContentHandler.ts new file mode 100644 index 000000000..d3f3b1595 --- /dev/null +++ b/packages/express-partial-content/src/createPartialContentHandler.ts @@ -0,0 +1,60 @@ +import type { Request, Response } from "express"; +import { parseRangeHeader } from "./parseRangeHeader.js"; +import { RangeParserError } from "./RangeParserError.js"; +import type { Logger } from "./Logger.js"; +import type { ContentProvider } from "./ContentProvider.js"; +import { ContentDoesNotExistError } from "./ContentDoesNotExistError.js"; +import { + getRangeHeader, + setContentRangeHeader, + setContentTypeHeader, + setContentDispositionHeader, + setAcceptRangesHeader, + setContentLengthHeader, + setCacheControlHeaderNoCache +} from "./utils.js"; +export function createPartialContentHandler(contentProvider: ContentProvider, logger: Logger) { + return async function handler(req: Request, res: Response) { + let content; + try { + content = await contentProvider(req); + } catch (error) { + logger.debug("createPartialContentHandler: ContentProvider threw exception: ", error); + if (error instanceof ContentDoesNotExistError) { + return res.status(404).send(error.message); + } + return res.sendStatus(500); + } + let { getStream, mimeType, fileName, totalSize } = content; + const rangeHeader = getRangeHeader(req); + let range; + try { + range = parseRangeHeader(rangeHeader, totalSize, logger); + } catch (error) { + logger.debug(`createPartialContentHandler: parseRangeHeader error: `, error); + if (error instanceof RangeParserError) { + setContentRangeHeader(null, totalSize, res); + return res.status(416).send(`Invalid value for Range: ${rangeHeader}`); + } + return res.sendStatus(500); + } + setContentTypeHeader(mimeType, res); + setContentDispositionHeader(fileName, res); + setAcceptRangesHeader(res); + // If range is not specified, or the file is empty, return the full stream + if (range === null) { + logger.debug("createPartialContentHandler: No range found, returning full content."); + setContentLengthHeader(String(totalSize), res); + return getStream().pipe(res); + } + setContentRangeHeader(range, totalSize, res); + let { start, end } = range; + setContentLengthHeader(String(start === end ? 0 : end - start + 1), res); + setCacheControlHeaderNoCache(res); + // Return 206 Partial Content status + logger.debug("createPartialContentHandler: Returning partial content for range: ", JSON.stringify(range)); + res.status(206); + + return getStream(range).pipe(res); + }; +} diff --git a/packages/express-partial-content/src/index.ts b/packages/express-partial-content/src/index.ts new file mode 100644 index 000000000..445d6027f --- /dev/null +++ b/packages/express-partial-content/src/index.ts @@ -0,0 +1,6 @@ +export * from "./Content.js"; +export * from "./ContentDoesNotExistError.js"; +export * from "./ContentProvider.js"; +export * from "./createPartialContentHandler.js"; +export * from "./Logger.js"; +export * from "./Range.js"; diff --git a/packages/express-partial-content/src/parseRangeHeader.spec.ts b/packages/express-partial-content/src/parseRangeHeader.spec.ts new file mode 100644 index 000000000..2bfde1961 --- /dev/null +++ b/packages/express-partial-content/src/parseRangeHeader.spec.ts @@ -0,0 +1,54 @@ +import { parseRangeHeader } from "./parseRangeHeader.js"; +import type { Logger } from "./Logger.js"; +import { RangeParserError } from "./RangeParserError.js"; + +describe("parseRangeHeader tests", () => { + let logger: Logger; + beforeEach(() => { + logger = { + debug: vi.fn() as (message: string, extra?: any) => void + }; + }); + afterEach(() => { + vi.restoreAllMocks(); + }); + it("returns null if range is not specified", () => { + let value = parseRangeHeader("", 10, logger); + expect(value).to.be.equal(null); + value = parseRangeHeader(null, 10, logger); + expect(value).to.be.equal(null); + }); + it("returns null if total size is zero", () => { + let value = parseRangeHeader("bytes=0-5", 0, logger); + expect(value).to.be.equal(null); + }); + it("if end is not provided, sets end to the last byte (totalSize - 1).", () => { + let value = parseRangeHeader("bytes=0-", 10, logger); + expect(value).to.be.deep.equal({ start: 0, end: 9 }); + }); + it('if start is not provided, set it to the offset of last "end" bytes from the end of the file.', () => { + let value = parseRangeHeader("bytes=-5", 10, logger); + expect(value).to.be.deep.equal({ start: 5, end: 9 }); + }); + it("handles invalid ranges", () => { + try { + parseRangeHeader("bytes=6-5", 10, logger); + } catch (error) { + expect(error).that.be.instanceOf(RangeParserError); + } + try { + parseRangeHeader("bytes=6-7", 10, logger); + } catch (error) { + expect(error).that.be.instanceOf(RangeParserError); + } + try { + parseRangeHeader("bytes=6-11", 10, logger); + } catch (error) { + expect(error).that.be.instanceOf(RangeParserError); + } + }); + it("returns a valid parsed range.", () => { + let value = parseRangeHeader("bytes=0-5", 10, logger); + expect(value).to.be.deep.equal({ start: 0, end: 5 }); + }); +}); diff --git a/packages/express-partial-content/src/parseRangeHeader.ts b/packages/express-partial-content/src/parseRangeHeader.ts new file mode 100644 index 000000000..189b932f4 --- /dev/null +++ b/packages/express-partial-content/src/parseRangeHeader.ts @@ -0,0 +1,65 @@ +import type { Logger } from "./Logger.js"; +import { RangeParserError } from "./RangeParserError.js"; +import type { Range } from "./Range.js"; + +const rangeRegEx = /bytes=([0-9]*)-([0-9]*)/; + +export function parseRangeHeader(_range: string | string[] | null | undefined, totalSize: number, logger: Logger): Range | null { + logger.debug("Un-parsed range is: ", _range); + + if (!_range?.length) { + return null; + } + + // TODO: Maybe we need to support multiple ranges. + const range = Array.isArray(_range) ? _range[0] : _range; + + // 1. If range is not specified or the file is empty, return null. + if (!range || range === null || range.length === 0 || totalSize === 0) { + return null; + } + + const splitRange = range.split(rangeRegEx); + const [, startValue, endValue] = splitRange; + let start = Number.parseInt(startValue); + let end = Number.parseInt(endValue); + + // 2. Parse start and end values and ensure they are within limits. + // 2.1. start: >= 0. + // 2.2. end: >= 0, <= totalSize - 1 + + let result = { + start: Number.isNaN(start) ? 0 : Math.max(start, 0), + end: Number.isNaN(end) ? totalSize - 1 : Math.min(Math.max(end, 0), totalSize - 1) + }; + + // 3.1. If end is not provided, set end to the last byte (totalSize - 1). + if (!Number.isNaN(start) && Number.isNaN(end)) { + logger.debug("End is not provided."); + + result.start = start; + result.end = totalSize - 1; + } + + // 3.2. If start is not provided, set it to the offset of last "end" bytes from the end of the file. + // And set end to the last byte. + // This way we return the last "end" bytes. + if (Number.isNaN(start) && !Number.isNaN(end)) { + logger.debug(`Start is not provided, "end" will be treated as last "end" bytes of the content.`); + + result.start = Math.max(totalSize - end, 0); + result.end = totalSize - 1; + } + + // 4. Handle invalid ranges. + if (start < 0 || start > end || end > totalSize) { + throw new RangeParserError(start, end); + } + + logRange(logger, result); + return result; +} + +function logRange(logger: Logger, range: Range) { + logger.debug("Range is: ", JSON.stringify(range)); +} diff --git a/packages/express-partial-content/src/utils.spec.ts b/packages/express-partial-content/src/utils.spec.ts new file mode 100644 index 000000000..3182e9af5 --- /dev/null +++ b/packages/express-partial-content/src/utils.spec.ts @@ -0,0 +1,111 @@ +import type { Request, Response } from "express"; +import { expect, type Mock } from "vitest"; + +import { + getHeader, + setHeader, + getRangeHeader, + setContentTypeHeader, + setContentLengthHeader, + setAcceptRangesHeader, + setContentDispositionHeader, + setContentRangeHeader, + setCacheControlHeaderNoCache +} from "./utils.js"; + +describe("utils tests", () => { + let req: Request; + let res: Response; + beforeEach(() => { + req = { + headers: { + "content-type": "application/octet-stream", + range: "*" + } + } as Request; + res = { + setHeader: vi.fn() as (name: string, value: string) => void + } as Response; + }); + describe("getHeader tests", () => { + it("gets the specified header value if present", () => { + const value = getHeader("content-type", req); + expect(value).to.equal("application/octet-stream"); + }); + it("returns undefined if the specified header value is absent", () => { + const value = getHeader("mime-type", req); + expect(value).to.be.undefined; + }); + }); + describe("setHeader tests", () => { + it("invokes res.setHeader API with the specified name and value args", () => { + const name = "Content-Type"; + const value = "application/octet-stream"; + setHeader(name, value, res); + expect(res.setHeader).toHaveBeenCalledExactlyOnceWith(name, value); + }); + }); + describe("getRangeHeader tests", () => { + it("gets range header value", () => { + const value = getRangeHeader(req); + expect(value).to.equal("*"); + }); + }); + describe("setContentTypeHeader tests", () => { + it("sets Content-Type header with specified value", () => { + const value = "application/octet-stream"; + setContentTypeHeader(value, res); + expect(res.setHeader).toHaveBeenCalledExactlyOnceWith("Content-Type", value); + }); + }); + describe("setContentLengthHeader tests", () => { + it("sets Content-Length header with specified value", () => { + const value = "100"; + setContentLengthHeader(value, res); + expect(res.setHeader).toHaveBeenCalledExactlyOnceWith("Content-Length", value); + }); + }); + describe("setAcceptRangesHeader tests", () => { + it("sets Accept-Ranges header with specified value", () => { + const value = "bytes"; + setAcceptRangesHeader(res); + expect(res.setHeader).toHaveBeenCalledExactlyOnceWith("Accept-Ranges", value); + }); + }); + describe("setContentRangeHeader tests", () => { + it("sets Content-Range header with specified value", () => { + let range: { start: number, end: number } | null = { start: 10, end: 100 }; + const size = 1000; + let value = `bytes ${range.start}-${range.end}/${size}`; + setContentRangeHeader(range, size, res); + expect(res.setHeader).toHaveBeenCalledExactlyOnceWith("Content-Range", value); + (res.setHeader as Mock).mockReset(); + + range = null; + value = `bytes */${size}`; + setContentRangeHeader(range, size, res); + expect(res.setHeader).toHaveBeenCalledExactlyOnceWith("Content-Range", value); + }); + }); + describe("setContentDispositionHeader tests", () => { + it("sets Content-Disposition header with specified value", () => { + const fileName = "file.txt"; + const value = `attachment; filename*=utf-8''${fileName}`; + setContentDispositionHeader(fileName, res); + expect(res.setHeader).toHaveBeenCalledExactlyOnceWith("Content-Disposition", value); + }); + it("sets Content-Disposition header with specified unicode", () => { + const fileName = "file.txt"; + const value = `attachment; filename*=utf-8''${encodeURIComponent(fileName)}`; + setContentDispositionHeader(fileName, res); + expect(res.setHeader).toHaveBeenCalledExactlyOnceWith("Content-Disposition", value); + }); + }); + describe("setCacheControlHeaderNoCache tests", () => { + it("sets Cache-Control header with specified value", () => { + const value = "no-cache"; + setCacheControlHeaderNoCache(res); + expect(res.setHeader).toHaveBeenCalledExactlyOnceWith("Cache-Control", value); + }); + }); +}); diff --git a/packages/express-partial-content/src/utils.ts b/packages/express-partial-content/src/utils.ts new file mode 100644 index 000000000..fc413362e --- /dev/null +++ b/packages/express-partial-content/src/utils.ts @@ -0,0 +1,13 @@ +import type { Request, Response } from "express"; +import type { Range } from "./Range.js"; +export const getHeader = (name: string, req: Request) => req.headers[name]; +export const getRangeHeader: (req: Request) => string | string[] | undefined = getHeader.bind(null, "range"); +export const setHeader = (name: string, value: string, res: Response) => res.setHeader(name, value); +export const setContentTypeHeader = setHeader.bind(null, "Content-Type"); +export const setContentLengthHeader = setHeader.bind(null, "Content-Length"); +export const setAcceptRangesHeader = setHeader.bind(null, "Accept-Ranges", "bytes"); +export const setContentRangeHeader = (range: Range | null, size: number, res: Response) => + setHeader("Content-Range", `bytes ${range ? `${range.start}-${range.end}` : "*"}/${size}`, res); +export const setContentDispositionHeader = (fileName: string, res: Response) => + setHeader("Content-Disposition", `attachment; filename*=utf-8''${encodeURIComponent(fileName)}`, res); +export const setCacheControlHeaderNoCache = setHeader.bind(null, "Cache-Control", "no-cache"); diff --git a/packages/express-partial-content/tsconfig.json b/packages/express-partial-content/tsconfig.json new file mode 100644 index 000000000..62ebbd946 --- /dev/null +++ b/packages/express-partial-content/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../tsconfig.base.json", + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + }, + { + "path": "./tsconfig.spec.json" + } + ] +} diff --git a/packages/express-partial-content/tsconfig.lib.json b/packages/express-partial-content/tsconfig.lib.json new file mode 100644 index 000000000..3872b6af8 --- /dev/null +++ b/packages/express-partial-content/tsconfig.lib.json @@ -0,0 +1,31 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "baseUrl": ".", + "rootDir": "src", + "outDir": "dist", + "tsBuildInfoFile": "dist/tsconfig.lib.tsbuildinfo", + "emitDeclarationOnly": false, + "types": [ + "node" + ] + }, + "include": [ + "src/**/*.ts" + ], + "references": [], + "exclude": [ + "vite.config.ts", + "vite.config.mts", + "vitest.config.ts", + "vitest.config.mts", + "src/**/*.test.ts", + "src/**/*.spec.ts", + "src/**/*.test.tsx", + "src/**/*.spec.tsx", + "src/**/*.test.js", + "src/**/*.spec.js", + "src/**/*.test.jsx", + "src/**/*.spec.jsx" + ] +} diff --git a/packages/express-partial-content/tsconfig.spec.json b/packages/express-partial-content/tsconfig.spec.json new file mode 100644 index 000000000..51c5a98fc --- /dev/null +++ b/packages/express-partial-content/tsconfig.spec.json @@ -0,0 +1,29 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./out-tsc/vitest", + "types": [ + "vitest/globals", + "vitest/importMeta", + "vite/client", + "node", + "vitest" + ] + }, + "include": [ + "vite.config.ts", + "vite.config.mts", + "vitest.config.ts", + "vitest.config.mts", + "src/**/*.test.ts", + "src/**/*.spec.ts", + "src/**/*.test.tsx", + "src/**/*.spec.tsx", + "src/**/*.test.js", + "src/**/*.spec.js", + "src/**/*.test.jsx", + "src/**/*.spec.jsx", + "src/**/*.d.ts", + "src/**/*.ts" + ] +} diff --git a/packages/express-partial-content/vite.config.ts b/packages/express-partial-content/vite.config.ts new file mode 100644 index 000000000..450381921 --- /dev/null +++ b/packages/express-partial-content/vite.config.ts @@ -0,0 +1,23 @@ +/// +import { defineConfig } from 'vite'; + +export default defineConfig(() => ({ + root: __dirname, + cacheDir: '../../node_modules/.vite/packages/express-partial-content', + plugins: [], + // Uncomment this if you are using workers. + // worker: { + // plugins: [ nxViteTsPaths() ], + // }, + test: { + watch: false, + globals: true, + environment: 'jsdom', + include: ['{src,tests}/**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}'], + reporters: ['default'], + coverage: { + reportsDirectory: './test-output/vitest/coverage', + provider: 'v8' as const, + } + }, +})); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index da58ea72b..ccb674518 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -444,8 +444,8 @@ importers: specifier: workspace:* version: link:../../packages/commons '@triliumnext/express-partial-content': - specifier: 1.0.1 - version: 1.0.1(express@4.21.2) + specifier: workspace:* + version: link:../../packages/express-partial-content '@triliumnext/turndown-plugin-gfm': specifier: workspace:* version: link:../../packages/turndown-plugin-gfm @@ -777,6 +777,12 @@ importers: specifier: ~0.5.11 version: 0.5.17 + packages/express-partial-content: + dependencies: + tslib: + specifier: ^2.3.0 + version: 2.8.1 + packages/turndown-plugin-gfm: dependencies: '@swc/helpers': @@ -3263,11 +3269,6 @@ packages: resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==} engines: {node: '>= 10'} - '@triliumnext/express-partial-content@1.0.1': - resolution: {integrity: sha512-WQipyCd3AHnKRhkMGUjgvi75L8kRjCyvCituvtdkNt5GLUQeHRyMwEokYJ8uoBgPoJNPLPCjXV1Ig0WE37N7KA==} - peerDependencies: - express: ^4.16.4 - '@trysound/sax@0.2.0': resolution: {integrity: sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==} engines: {node: '>=10.13.0'} @@ -14048,10 +14049,6 @@ snapshots: '@tootallnate/once@2.0.0': {} - '@triliumnext/express-partial-content@1.0.1(express@4.21.2)': - dependencies: - express: 4.21.2 - '@trysound/sax@0.2.0': {} '@tweenjs/tween.js@25.0.0': {} diff --git a/tsconfig.json b/tsconfig.json index 0c1b0f48f..094d2a3fc 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -29,6 +29,9 @@ }, { "path": "./apps/edit-docs" + }, + { + "path": "./packages/express-partial-content" } ] }