mynode/packages/app-cli/tests/HtmlToMd.ts

import shim from '@joplin/lib/shim';
import * as os from 'os';
import { readFile } from 'fs/promises';
import { filename } from '@joplin/lib/path-utils';
import { dirname } from 'path';
import HtmlToMd, { ParseOptions } from '@joplin/lib/HtmlToMd';
import MdToHtml from '@joplin/renderer/MdToHtml';

describe('HtmlToMd', () => {

	it('should convert from Html to Markdown', (async () => {
		const basePath = `${__dirname}/html_to_md`;
		const files = await shim.fsDriver().readDirStats(basePath);
		const htmlToMd = new HtmlToMd();

		for (let i = 0; i < files.length; i++) {
			const htmlFilename = files[i].path;
			if (htmlFilename.indexOf('.html') < 0) continue;

			const htmlPath = `${basePath}/${htmlFilename}`;
			const mdPath = `${basePath}/${filename(htmlFilename)}.md`;

			// if (htmlFilename !== 'anchor_same_title_and_url.html') continue;

			// if (htmlFilename.indexOf('image_preserve_size') !== 0) continue;

			const htmlToMdOptions: ParseOptions = {};

			if (htmlFilename === 'anchor_local.html') {
				// Normally the list of anchor names in the document are retrieved from the HTML code
				// This is straightforward when the document is still in DOM format, as with the clipper,
				// but otherwise it would need to be somehow parsed out from the HTML. Here we just
				// hard code the anchors that we know are in the file.
				htmlToMdOptions.anchorNames = ['first', 'second', 'fourth'];
			}

			if (htmlFilename.indexOf('image_preserve_size') === 0) {
				htmlToMdOptions.preserveImageTagsWithSize = true;
			}

			if (htmlFilename.indexOf('preserve_nested_tables') === 0) {
				htmlToMdOptions.preserveNestedTables = true;
			}

			if (htmlFilename.indexOf('text_color') === 0) {
				htmlToMdOptions.preserveColorStyles = true;
			}

			if (htmlFilename.indexOf('table_with') === 0 || htmlFilename.indexOf('table_default') === 0) {
				htmlToMdOptions.preserveTableStyles = true;
			}

			const html = await readFile(htmlPath, 'utf8');
			let expectedMd = await readFile(mdPath, 'utf8');

			let actualMd = await htmlToMd.parse(`<div>${html}</div>`, htmlToMdOptions);

			if (os.EOL === '\r\n') {
				expectedMd = expectedMd.replace(/\r\n/g, '\n');
				actualMd = actualMd.replace(/\r\n/g, '\n');
			}

			if (actualMd !== expectedMd) {
				const result = [];
				result.push('');
				result.push(`Error converting file: ${htmlFilename}`);
				result.push('--------------------------------- Got:');
				result.push(actualMd.split('\n').map((l: string) => `"${l}"`).join('\n'));
				// result.push('--------------------------------- Raw:');
				// result.push(actualMd.split('\n'));
				result.push('--------------------------------- Expected:');
				result.push(expectedMd.split('\n').map((l: string) => `"${l}"`).join('\n'));
				result.push('--------------------------------------------');
				result.push('');

				// eslint-disable-next-line no-console
				console.info(result.join('\n'));

				// console.info('');
				// console.info(`Error converting file: ${htmlFilename}`);
				// console.info('--------------------------------- Got:');
				// console.info(actualMd);
				// console.info('--------------------------------- Raw:');
				// console.info(actualMd.split('\n'));
				// console.info('--------------------------------- Expected:');
				// console.info(expectedMd.split('\n'));
				// console.info('--------------------------------------------');
				// console.info('');

				expect(false).toBe(true);
				// return;
			} else {
				expect(true).toBe(true);
			}
		}
	}));

	it('should allow disabling escape', async () => {
		const htmlToMd = new HtmlToMd();
		expect(htmlToMd.parse('> 1 _2_ 3.pdf', { disableEscapeContent: true })).toBe('> 1 _2_ 3.pdf');
		expect(htmlToMd.parse('> 1 _2_ 3.pdf', { disableEscapeContent: false })).toBe('\\> 1 \\_2_ 3.pdf');
	});

	it('should support tightLists option', async () => {
		const htmlToMd = new HtmlToMd();
		const html = '<ul><li><p><strong>Item 1</strong></p></li><li><p><strong>Item 2</strong></p></li><li><p><strong>Item 3</strong></p></li></ul>';

		// Without tightLists, paragraphs inside list items produce extra blank lines
		const looseResult = htmlToMd.parse(html, { tightLists: false });
		expect(looseResult).toContain('\n    \n');

		// With tightLists, list items are compact without blank lines
		const tightResult = htmlToMd.parse(html, { tightLists: true });
		expect(tightResult).toBe('- **Item 1**\n- **Item 2**\n- **Item 3**');
	});

	it('should support collapseMultipleBlankLines option', async () => {
		const htmlToMd = new HtmlToMd();
		const html = '<p>First</p><br><br><br><p>Second</p>';

		// Without collapseMultipleBlankLines, multiple blank lines are preserved
		const looseResult = htmlToMd.parse(html, { collapseMultipleBlankLines: false });
		expect(looseResult).toContain('\n\n  \n');

		// With collapseMultipleBlankLines, multiple blank lines are collapsed into one
		const collapsedResult = htmlToMd.parse(html, { collapseMultipleBlankLines: true });
		expect(collapsedResult).not.toContain('\n\n\n');
		expect(collapsedResult).not.toContain('\n\n  \n');

		// Verify that a single blank line is preserved (not fully removed)
		expect(collapsedResult).toContain('\n\n');
	});

	it('should not re-read the rows of a table for every row', async () => {
		// Under domino, each access to `table.rows` walks the whole table, so reading it once per row made
		// converting large tables quadratic: https://github.com/laurent22/joplin/issues/16894
		const resolveFrom = (moduleName: string, fromPath: string) => require.resolve(moduleName, { paths: [dirname(fromPath)] });
		const domino: { createDocument: (html: string)=> Document } = require(resolveFrom('@mixmark-io/domino', resolveFrom('@joplin/turndown', require.resolve('@joplin/lib/HtmlToMd'))));

		const rowCount = 100;
		const rows = Array.from({ length: rowCount }, (_, i) => `<tr><td>${i}</td><td>b</td><td>c</td></tr>`).join('');
		const root = domino.createDocument(`<div><table><thead><tr><th>A</th><th>B</th><th>C</th></tr></thead><tbody>${rows}</tbody></table></div>`).querySelector('div');
		// Turndown converts a clone of the element it's given, which wouldn't have the counter below
		Object.defineProperty(root, 'cloneNode', { value: () => root });

		const table = root.querySelector('table');
		const getRows = Object.getOwnPropertyDescriptor(Object.getPrototypeOf(table), 'rows').get;
		let rowsReadCount = 0;
		Object.defineProperty(table, 'rows', { get: () => {
			rowsReadCount++;
			return getRows.call(table);
		} });

		const markdown = new HtmlToMd().parse(root);
		expect(markdown.split('\n')).toHaveLength(rowCount + 2);
		expect(rowsReadCount).toBeLessThan(rowCount);
	});

	// Covers how newlines are carried between children when joining them, which was rewritten to fix
	// https://github.com/laurent22/joplin/issues/16895
	it.each([
		['<ul><li>a</li><li></li><li>b</li></ul>', '- a\n\n- b'],
		['<span><p>a</p></span>b', 'a\n\nb'],
		['<pre><code><span>line 1</span>\n\n\n<span>line 2</span></code></pre>', '```\nline 1\n\n\nline 2\n```'],
	])('should join the Markdown of child elements: %j', (html, expected) => {
		expect(new HtmlToMd().parse(html)).toBe(expected);
	});

	it.each([
		['inline code ending with a space at the end of a heading', '<h2><code>npm</code> install&nbsp;<code>a&nbsp;</code></h2>', 'a\u00A0'],
		['a heading that only contains inline code with a space', '<h2><code>&nbsp;</code></h2>', '\u00A0'],
	])('should not change %s when the note is saved again', async (_title, html, codeText) => {
		const htmlToMd = new HtmlToMd();
		const mdToHtml = new MdToHtml();

		const firstSave = htmlToMd.parse(html);
		const reopened = await mdToHtml.render(firstSave, null, { bodyOnly: true });
		const secondSave = htmlToMd.parse(reopened.html);

		expect(reopened.html).toContain(`>${codeText}</code>`);
		expect(secondSave).toBe(firstSave);
	});

});