diff --git a/packages/turndown-plugin-gfm/README.md b/packages/turndown-plugin-gfm/README.md index 528f910c4..b9f1ab294 100644 --- a/packages/turndown-plugin-gfm/README.md +++ b/packages/turndown-plugin-gfm/README.md @@ -1,13 +1,5 @@ # turndown-plugin-gfm -* * * - -**IMPORTANT: This package is now hosted there: https://github.com/laurent22/joplin/tree/dev/packages/turndown-plugin-gfm** - -It can be installed using `npm install --save @joplin/turndown-plugin-gfm` - -* * * - A [Turndown](https://github.com/domchristie/turndown) plugin which adds GitHub Flavored Markdown extensions. This is a fork of the original [turndown-plugin-gfm](https://github.com/domchristie/turndown-plugin-gfm) for use with [Joplin](https://github.com/laurent22/joplin). The changes are: @@ -25,15 +17,15 @@ This is a fork of the original [turndown-plugin-gfm](https://github.com/domchris npm: ``` -npm install joplin-turndown-plugin-gfm +npm install @joplin/turndown-plugin-gfm ``` ## Usage ```js // For Node.js -var TurndownService = require('turndown') -var turndownPluginGfm = require('joplin-turndown-plugin-gfm') +var TurndownService = require('@joplin/turndown') +var turndownPluginGfm = require('@joplin/turndown-plugin-gfm') var gfm = turndownPluginGfm.gfm var turndownService = new TurndownService() @@ -51,11 +43,22 @@ turndown-plugin-gfm is a suite of plugins which can be applied individually. The So for example, if you only wish to convert tables: ```js -var tables = require('turndown-plugin-gfm').tables +var tables = require('@joplin/turndown-plugin-gfm').tables var turndownService = new TurndownService() turndownService.use(tables) ``` +### Typescript + +To use this in a typescript project, add this to a `declarations.d.ts` file, as described in https://www.npmjs.com/package/@joplin/turndown, and then add: + +```ts +declare module "@joplin/turndown-plugin-gfm" { + export const gfm: any; + // Add other named exports if necessary +} +``` + ## License turndown-plugin-gfm is copyright © 2017+ Dom Christie and released under the MIT license. diff --git a/packages/turndown-plugin-gfm/build_for_test.sh b/packages/turndown-plugin-gfm/build_for_test.sh index abdf1c4d7..9cf93ad68 100755 --- a/packages/turndown-plugin-gfm/build_for_test.sh +++ b/packages/turndown-plugin-gfm/build_for_test.sh @@ -1,10 +1,8 @@ #!/bin/bash -ROOT_DIR=/mnt/c/Users/laurent/src/joplin -CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" set -e + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +ROOT_DIR="$SCRIPT_DIR/../.." + npm run build -# rsync -a ./dist/ $ROOT_DIR/ElectronClient/app/node_modules/joplin-turndown-plugin-gfm/dist/ -# rsync -a ./lib/ $ROOT_DIR/ElectronClient/app/node_modules/joplin-turndown-plugin-gfm/lib/ -rm -rf $ROOT_DIR/CliClient/node_modules/joplin-turndown-plugin-gfm -ln -s "$CURRENT_DIR" $ROOT_DIR/CliClient/node_modules/joplin-turndown-plugin-gfm -$ROOT_DIR/CliClient/run_test.sh HtmlToMd \ No newline at end of file +cd $ROOT_DIR/packages/app-cli && npm run test -- HtmlToMd \ No newline at end of file diff --git a/packages/turndown-plugin-gfm/config/rollup.config.browser.cjs.js b/packages/turndown-plugin-gfm/config/rollup.config.browser.cjs.js index 3a300e6ee..da106af42 100644 --- a/packages/turndown-plugin-gfm/config/rollup.config.browser.cjs.js +++ b/packages/turndown-plugin-gfm/config/rollup.config.browser.cjs.js @@ -1,8 +1,8 @@ -import config from './rollup.config' +import config from './rollup.config'; export default config({ - output: { - format: 'cjs', - file: 'lib/turndown-plugin-gfm.browser.cjs.js' - } -}) + output: { + format: 'cjs', + file: 'lib/turndown-plugin-gfm.browser.cjs.js', + }, +}); diff --git a/packages/turndown-plugin-gfm/config/rollup.config.browser.es.js b/packages/turndown-plugin-gfm/config/rollup.config.browser.es.js index 0e00ddeb1..595d72caf 100644 --- a/packages/turndown-plugin-gfm/config/rollup.config.browser.es.js +++ b/packages/turndown-plugin-gfm/config/rollup.config.browser.es.js @@ -1,8 +1,8 @@ -import config from './rollup.config' +import config from './rollup.config'; export default config({ - output: { - format: 'es', - file: 'lib/turndown-plugin-gfm.browser.es.js' - } -}) + output: { + format: 'es', + file: 'lib/turndown-plugin-gfm.browser.es.js', + }, +}); diff --git a/packages/turndown-plugin-gfm/config/rollup.config.cjs.js b/packages/turndown-plugin-gfm/config/rollup.config.cjs.js index ddbf270c6..892b93273 100644 --- a/packages/turndown-plugin-gfm/config/rollup.config.cjs.js +++ b/packages/turndown-plugin-gfm/config/rollup.config.cjs.js @@ -1,8 +1,8 @@ -import config from './rollup.config' +import config from './rollup.config'; export default config({ - output: { - format: 'cjs', - file: 'lib/turndown-plugin-gfm.cjs.js' - } -}) + output: { + format: 'cjs', + file: 'lib/turndown-plugin-gfm.cjs.js', + }, +}); diff --git a/packages/turndown-plugin-gfm/config/rollup.config.es.js b/packages/turndown-plugin-gfm/config/rollup.config.es.js index 7370da61f..e3c2663f0 100644 --- a/packages/turndown-plugin-gfm/config/rollup.config.es.js +++ b/packages/turndown-plugin-gfm/config/rollup.config.es.js @@ -1,8 +1,8 @@ -import config from './rollup.config' +import config from './rollup.config'; export default config({ - output: { - format: 'es', - file: 'lib/turndown-plugin-gfm.es.js' - } -}) + output: { + format: 'es', + file: 'lib/turndown-plugin-gfm.es.js', + }, +}); diff --git a/packages/turndown-plugin-gfm/config/rollup.config.iife.js b/packages/turndown-plugin-gfm/config/rollup.config.iife.js index 8dafb4fe2..a3e73e994 100644 --- a/packages/turndown-plugin-gfm/config/rollup.config.iife.js +++ b/packages/turndown-plugin-gfm/config/rollup.config.iife.js @@ -1,8 +1,8 @@ -import config from './rollup.config' +import config from './rollup.config'; export default config({ - output: { - format: 'iife', - file: 'dist/turndown-plugin-gfm.js' - } -}) + output: { + format: 'iife', + file: 'dist/turndown-plugin-gfm.js', + }, +}); diff --git a/packages/turndown-plugin-gfm/config/rollup.config.js b/packages/turndown-plugin-gfm/config/rollup.config.js index b8ec6adae..ce748beba 100644 --- a/packages/turndown-plugin-gfm/config/rollup.config.js +++ b/packages/turndown-plugin-gfm/config/rollup.config.js @@ -1,7 +1,7 @@ -export default function (config) { - return { - name: 'turndownPluginGfm', - input: 'src/gfm.js', - output: config.output - } +export default function(config) { + return { + name: 'turndownPluginGfm', + input: 'src/gfm.js', + output: config.output, + }; } diff --git a/packages/turndown-plugin-gfm/package.json b/packages/turndown-plugin-gfm/package.json index e7401fd1d..758485d6c 100644 --- a/packages/turndown-plugin-gfm/package.json +++ b/packages/turndown-plugin-gfm/package.json @@ -1,17 +1,18 @@ { - "name": "joplin-turndown-plugin-gfm", + "name": "@joplin/turndown-plugin-gfm", "description": "Turndown plugin to add GitHub Flavored Markdown extensions.", - "version": "1.0.12", + "publishConfig": { + "access": "public" + }, + "version": "1.0.61", "author": "Dom Christie", "main": "lib/turndown-plugin-gfm.cjs.js", - "module": "lib/turndown-plugin-gfm.es.js", - "jsnext:main": "lib/turndown-plugin-gfm.es.js", "devDependencies": { - "browserify": "^14.5.0", - "rollup": "^0.50.0", - "standard": "^10.0.3", - "turndown": "4.0.1", - "turndown-attendant": "0.0.2" + "browserify": "14.5.0", + "rollup": "0.50.1", + "standard": "17.1.0", + "turndown": "7.2.0", + "turndown-attendant": "0.0.3" }, "files": [ "lib", @@ -32,12 +33,13 @@ "url": "https://github.com/laurent22/joplin-turndown-plugin-gfm.git" }, "scripts": { - "build": "npm run build-cjs && npm run build-es && npm run build-iife", + "build-all": "npm run build-cjs && npm run build-es && npm run build-iife", + "build": "rollup -c config/rollup.config.cjs.js", "build-cjs": "rollup -c config/rollup.config.cjs.js && rollup -c config/rollup.config.browser.cjs.js", "build-es": "rollup -c config/rollup.config.es.js && rollup -c config/rollup.config.browser.es.js", "build-iife": "rollup -c config/rollup.config.iife.js", "build-test": "browserify test/turndown-plugin-gfm-test.js --outfile test/turndown-plugin-gfm-test.browser.js", - "prepublish": "npm run build", - "test": "npm run build && standard ./src/**/*.js && node test/turndown-plugin-gfm-test.js" - } + "prepare": "npm run build" + }, + "gitHead": "05a29b450962bf05a8642bbd39446a1f679a96ba" } diff --git a/packages/turndown-plugin-gfm/publish.sh b/packages/turndown-plugin-gfm/publish.sh old mode 100755 new mode 100644 diff --git a/packages/turndown-plugin-gfm/src/strikethrough.js b/packages/turndown-plugin-gfm/src/strikethrough.js index 5cbe40da5..9dfb9d7bb 100644 --- a/packages/turndown-plugin-gfm/src/strikethrough.js +++ b/packages/turndown-plugin-gfm/src/strikethrough.js @@ -2,7 +2,7 @@ export default function strikethrough (turndownService) { turndownService.addRule('strikethrough', { filter: ['del', 's', 'strike'], replacement: function (content) { - return '~' + content + '~' + return '~~' + content + '~~' } }) } diff --git a/packages/turndown-plugin-gfm/src/tables.js b/packages/turndown-plugin-gfm/src/tables.js index 74f5c526d..7bdc0ed36 100644 --- a/packages/turndown-plugin-gfm/src/tables.js +++ b/packages/turndown-plugin-gfm/src/tables.js @@ -1,6 +1,48 @@ var indexOf = Array.prototype.indexOf var every = Array.prototype.every var rules = {} +var alignMap = { left: ':---', right: '---:', center: ':---:' }; + +let isCodeBlock_ = null; +let options_ = null; + +// We need to cache the result of tableShouldBeSkipped() as it is expensive. +// Caching it means we went from about 9000 ms for rendering down to 90 ms. +// Fixes https://github.com/laurent22/joplin/issues/6736 +const tableShouldBeSkippedCache_ = new WeakMap(); + +function getAlignment(node) { + return node ? (node.getAttribute('align') || node.style.textAlign || '').toLowerCase() : ''; +} + +function getBorder(alignment) { + return alignment ? alignMap[alignment] : '---'; +} + +function getColumnAlignment(table, columnIndex) { + var votes = { + left: 0, + right: 0, + center: 0, + '': 0, + }; + + var align = ''; + + for (var i = 0; i < table.rows.length; ++i) { + var row = table.rows[i]; + if (columnIndex < row.childNodes.length) { + var cellAlignment = getAlignment(row.childNodes[columnIndex]); + ++votes[cellAlignment]; + + if (votes[cellAlignment] > votes[align]) { + align = cellAlignment; + } + } + } + + return align; +} rules.tableCell = { filter: ['th', 'td'], @@ -17,22 +59,13 @@ rules.tableRow = { if (tableShouldBeSkipped(parentTable)) return content; var borderCells = '' - var alignMap = { left: ':--', right: '--:', center: ':-:' } if (isHeadingRow(node)) { const colCount = tableColCount(parentTable); for (var i = 0; i < colCount; i++) { - const childNode = colCount >= node.childNodes.length ? null : node.childNodes[i]; - var border = '---' - var align = childNode ? (childNode.getAttribute('align') || '').toLowerCase() : ''; - - if (align) border = alignMap[align] || border - - if (childNode) { - borderCells += cell(border, node.childNodes[i]) - } else { - borderCells += cell(border, null, i); - } + const childNode = i < node.childNodes.length ? node.childNodes[i] : null; + var border = getBorder(getColumnAlignment(parentTable, i)); + borderCells += cell(border, childNode, i); } } return '\n' + content + (borderCells ? '\n' + borderCells : '') @@ -40,33 +73,70 @@ rules.tableRow = { } rules.table = { - // Only convert tables with a heading row. - // Tables with no heading row are kept using `keep` (see below). - filter: function (node) { - return node.nodeName === 'TABLE' + filter: function (node, options) { + return node.nodeName === 'TABLE'; }, replacement: function (content, node) { - if (tableShouldBeSkipped(node)) return content; + // Only convert tables that can result in valid Markdown + // Other tables are kept as HTML using `keep` (see below). + if (tableShouldBeHtml(node, options_)) { + let html = node.outerHTML; + let divParent = nodeParentDiv(node) + // Make table in HTML format horizontally scrollable by give table a div parent, so the width of the table is limited to the screen width. + // see https://github.com/laurent22/joplin/pull/10161 + // test cases: + // packages/app-cli/tests/html_to_md/preserve_nested_tables.html + // packages/app-cli/tests/html_to_md/table_with_blockquote.html + // packages/app-cli/tests/html_to_md/table_with_code_1.html + // packages/app-cli/tests/html_to_md/table_with_code_2.html + // packages/app-cli/tests/html_to_md/table_with_code_3.html + // packages/app-cli/tests/html_to_md/table_with_heading.html + // packages/app-cli/tests/html_to_md/table_with_hr.html + // packages/app-cli/tests/html_to_md/table_with_list.html + if (divParent === null || !divParent.classList.contains('joplin-table-wrapper')){ + return `\n\n
${html}
\n\n`; + } else { + return html + } + } else { + if (tableShouldBeSkipped(node)) return content; - // Ensure there are no blank lines - content = content.replace(/\n+/g, '\n') + // Ensure there are no blank lines + content = content.replace(/\n+/g, '\n') - // If table has no heading, add an empty one so as to get a valid Markdown table - var secondLine = content.trim().split('\n'); - if (secondLine.length >= 2) secondLine = secondLine[1] - var secondLineIsDivider = secondLine.indexOf('| ---') === 0 - - var columnCount = tableColCount(node); - var emptyHeader = '' - if (columnCount && !secondLineIsDivider) { - emptyHeader = '|' + ' |'.repeat(columnCount) + '\n' + '|' + ' --- |'.repeat(columnCount) + // If table has no heading, add an empty one so as to get a valid Markdown table + var secondLine = content.trim().split('\n'); + if (secondLine.length >= 2) secondLine = secondLine[1] + var secondLineIsDivider = /\| :?---/.test(secondLine); + + var columnCount = tableColCount(node); + var emptyHeader = '' + if (columnCount && !secondLineIsDivider) { + emptyHeader = '|' + ' |'.repeat(columnCount) + '\n' + '|' + for (var columnIndex = 0; columnIndex < columnCount; ++columnIndex) { + emptyHeader += ' ' + getBorder(getColumnAlignment(node, columnIndex)) + ' |'; + } + } + + const captionContent = node.caption ? node.caption.textContent || '' : ''; + const caption = captionContent ? `${captionContent}\n\n` : ''; + const tableContent = `${emptyHeader}${content}`.trimStart(); + return `\n\n${caption}${tableContent}\n\n`; } - - return '\n\n' + emptyHeader + content + '\n\n' } } +rules.tableCaption = { + filter: ['caption'], + replacement: () => '', +}; + +rules.tableColgroup = { + filter: ['colgroup', 'col'], + replacement: () => '', +}; + rules.tableSection = { filter: ['thead', 'tbody', 'tfoot'], replacement: function (content) { @@ -126,9 +196,57 @@ function nodeContainsTable(node) { return false; } +const nodeContains = (node, types) => { + if (!node.childNodes) return false; + + for (let i = 0; i < node.childNodes.length; i++) { + const child = node.childNodes[i]; + if (types === 'code' && isCodeBlock_ && isCodeBlock_(child)) return true; + if (types.includes(child.nodeName)) return true; + if (nodeContains(child, types)) return true; + } + + return false; +} + +const tableShouldBeHtml = (tableNode, options) => { + const possibleTags = [ + 'UL', + 'OL', + 'H1', + 'H2', + 'H3', + 'H4', + 'H5', + 'H6', + 'HR', + 'BLOCKQUOTE', + ]; + + // In general we should leave as HTML tables that include other tables. The + // exception is with the Web Clipper when we import a web page with a layout + // that's made of HTML tables. In that case we have this logic of removing the + // outer table and keeping only the inner ones. For the Rich Text editor + // however we always want to keep nested tables. + if (options.preserveNestedTables) possibleTags.push('TABLE'); + + return nodeContains(tableNode, 'code') || + nodeContains(tableNode, possibleTags); +} + // Various conditions under which a table should be skipped - i.e. each cell // will be rendered one after the other as if they were paragraphs. function tableShouldBeSkipped(tableNode) { + const cached = tableShouldBeSkippedCache_.get(tableNode); + if (cached !== undefined) return cached; + + const result = tableShouldBeSkipped_(tableNode); + + tableShouldBeSkippedCache_.set(tableNode, result); + return result; +} + +function tableShouldBeSkipped_(tableNode) { if (!tableNode) return true; if (!tableNode.rows) return true; if (tableNode.rows.length === 1 && tableNode.rows[0].childNodes.length <= 1) return true; // Table with only one cell @@ -136,6 +254,15 @@ function tableShouldBeSkipped(tableNode) { return false; } +function nodeParentDiv(node) { + let parent = node.parentNode; + while (parent.nodeName !== 'DIV') { + parent = parent.parentNode; + if (!parent) return null; + } + return parent; +} + function nodeParentTable(node) { let parent = node.parentNode; while (parent.nodeName !== 'TABLE') { @@ -164,8 +291,12 @@ function tableColCount(node) { } export default function tables (turndownService) { + isCodeBlock_ = turndownService.isCodeBlock; + options_ = turndownService.options; + turndownService.keep(function (node) { - return node.nodeName === 'TABLE' - }) + if (node.nodeName === 'TABLE' && tableShouldBeHtml(node, turndownService.options)) return true; + return false; + }); for (var key in rules) turndownService.addRule(key, rules[key]) } diff --git a/packages/turndown-plugin-gfm/test/turndown-plugin-gfm-test.js b/packages/turndown-plugin-gfm/test/turndown-plugin-gfm-test.js index 3a6cdb4ec..414dc6460 100644 --- a/packages/turndown-plugin-gfm/test/turndown-plugin-gfm-test.js +++ b/packages/turndown-plugin-gfm/test/turndown-plugin-gfm-test.js @@ -1,13 +1,13 @@ -var Attendant = require('turndown-attendant') -var TurndownService = require('turndown') -var gfm = require('../lib/turndown-plugin-gfm.cjs').gfm +const Attendant = require('turndown-attendant'); +const TurndownService = require('turndown'); +const gfm = require('../lib/turndown-plugin-gfm.cjs').gfm; -var attendant = new Attendant({ - file: __dirname + '/index.html', - TurndownService: TurndownService, - beforeEach: function (turndownService) { - turndownService.use(gfm) - } -}) +const attendant = new Attendant({ + file: `${__dirname}/index.html`, + TurndownService: TurndownService, + beforeEach: function(turndownService) { + turndownService.use(gfm); + }, +}); -attendant.run() +attendant.run();