feat: Initial clean release of CHM

packages/editor/CodeMirror/extensions/rendering/utils/makeBlockReplaceExtension.ts

@@ -0,0 +1,98 @@
+import { EditorView, Decoration, DecorationSet, WidgetType } from '@codemirror/view';
+import { syntaxTree } from '@codemirror/language';
+import { EditorState, Extension, Range, StateField } from '@codemirror/state';
+import { ReplacementExtension } from '../types';
+import nodeIntersectsSelection from './nodeIntersectsSelection';
+
+const updateDecorations = (state: EditorState, extensionSpec: ReplacementExtension) => {
+	const doc = state.doc;
+	const cursorLine = doc.lineAt(state.selection.main.anchor);
+
+	const parentTagCounts = new Map<string, number>();
+	const widgets: Range<Decoration>[] = [];
+	syntaxTree(state).iterate({
+		enter: node => {
+			parentTagCounts.set(node.name, (parentTagCounts.get(node.name) ?? 0) + 1);
+
+			const nodeLineFrom = doc.lineAt(node.from);
+			const nodeLineTo = doc.lineAt(node.to);
+			const selectionIsNearNode = Math.abs(nodeLineFrom.number - cursorLine.number) <= 1 || Math.abs(nodeLineTo.number - cursorLine.number) <= 1;
+			const shouldHide = (
+				(extensionSpec.hideWhenContainsSelection ?? true) && (
+					nodeIntersectsSelection(state.selection, node) || selectionIsNearNode
+				)
+			);
+
+			if (!shouldHide) {
+				const widget = extensionSpec.createDecoration(node, state, parentTagCounts);
+				if (widget) {
+					let decoration;
+					if (widget instanceof WidgetType) {
+						decoration = Decoration.replace({
+							widget,
+							block: true,
+						});
+					} else {
+						decoration = widget;
+					}
+
+					let rangeFrom = nodeLineFrom.from;
+					let rangeTo = nodeLineTo.to;
+					let skip = false;
+					if (extensionSpec.getDecorationRange) {
+						const range = extensionSpec.getDecorationRange(node, state, parentTagCounts);
+						if (range) {
+							rangeFrom = range[0];
+							rangeTo = range.length === 1 ? range[0] : range[1];
+						} else {
+							skip = true;
+						}
+					}
+
+					if (!skip) {
+						widgets.push(decoration.range(rangeFrom, rangeTo));
+					}
+				}
+			}
+		},
+		leave: node => {
+			parentTagCounts.set(node.name, (parentTagCounts.get(node.name) ?? 0) - 1);
+		},
+	});
+
+	return Decoration.set(widgets, true);
+};
+
+const makeBlockReplaceExtension = (extensionSpec: ReplacementExtension) => {
+	const blockDecorationField = StateField.define<DecorationSet>({
+		create(state) {
+			return updateDecorations(state, extensionSpec);
+		},
+		update(decorations, transaction) {
+			decorations = decorations.map(transaction.changes);
+			const selectionChanged = !transaction.newSelection.eq(transaction.startState.selection);
+
+			const wasRerenderRequested = () => {
+				if (!extensionSpec.shouldFullReRender) return false;
+				return extensionSpec.shouldFullReRender(transaction);
+			};
+
+			const treeChanged = syntaxTree(transaction.state) !== syntaxTree(transaction.startState);
+
+			if (transaction.docChanged || selectionChanged || wasRerenderRequested() || treeChanged) {
+				decorations = updateDecorations(transaction.state, extensionSpec);
+			}
+
+			return decorations;
+		},
+		provide: f => EditorView.decorations.from(f),
+	});
+	const extensions: Extension[] = [blockDecorationField];
+	if (extensionSpec.atomic) {
+		extensions.push(EditorView.atomicRanges.of(view => view.state.field(blockDecorationField)));
+	}
+	return extensions;
+};
+
+export default makeBlockReplaceExtension;
+

packages/editor/CodeMirror/extensions/rendering/utils/makeHideReplaceExtension.ts

@@ -0,0 +1,70 @@
+import makeInlineReplaceExtension from './makeInlineReplaceExtension';
+import { SyntaxNodeRef } from '@lezer/common';
+import { EditorState, Facet } from '@codemirror/state';
+import { Decoration, EditorView } from '@codemirror/view';
+import { ParentTags, ReplacementExtension } from '../types';
+
+const hideDecoration = Decoration.mark({
+	class: 'cm-hidden',
+});
+// Don't fully hide replaced Markdown:
+// - Hiding text with 'display: none' causes selection to behave unexpectedly
+// - Screen readers skip text with 'display: none', which can lead to a confusing editing experience
+const hiddenStyles = EditorView.theme({
+	['& .cm-hidden']: {
+		// This needs to be non-zero to avoid selection issues. To avoid screen reader issues
+		// the width also needs to be >= 1px.
+		width: '1px',
+		height: '1em',
+		marginRight: '-0.9px',
+
+		opacity: '0',
+		display: 'inline-block',
+		whiteSpace: 'nowrap',
+		overflow: 'hidden',
+
+		// Decrease the size of the content: Work around cursor placement issues when a
+		// line ends with a long hidden decoration:
+		transform: 'scale(0.1, 1)',
+	},
+});
+
+type Options = Pick<ReplacementExtension, 'getRevealStrategy'|'getDecorationRange'> & {
+	shouldHide: (node: SyntaxNodeRef, state: EditorState, parentTags: ParentTags)=> boolean;
+};
+
+const hideReplaceFacet = Facet.define<Options>({
+	combine: values => values.flat(),
+	enables: facet => {
+		const getActivePluginOptions = (node: SyntaxNodeRef, state: EditorState, parentTags: ParentTags) => {
+			const allPluginOptions = state.facet(facet);
+			return allPluginOptions.find(o => o.shouldHide(node, state, parentTags));
+		};
+		return [
+			hiddenStyles,
+			makeInlineReplaceExtension({
+				createDecoration: (node, state, parentTags) => {
+					if (getActivePluginOptions(node, state, parentTags)) {
+						return hideDecoration;
+					}
+					return null;
+				},
+				getDecorationRange: (node, state, parentTags) => {
+					const options = getActivePluginOptions(node, state, parentTags);
+					return options?.getDecorationRange?.(node, state, parentTags);
+				},
+				getRevealStrategy: (node, state, parentTags) => {
+					const options = getActivePluginOptions(node, state, parentTags);
+					return options?.getRevealStrategy?.(node, state, parentTags);
+				},
+				mergeNeighbors: true,
+			}),
+		];
+	},
+});
+
+const makeHideReplaceExtension = (options: Options) => [
+	hideReplaceFacet.of(options),
+];
+
+export default makeHideReplaceExtension;

packages/editor/CodeMirror/extensions/rendering/utils/makeInlineReplaceExtension.test.ts

@@ -0,0 +1,213 @@
+import { EditorSelection, Extension } from '@codemirror/state';
+import createTestEditor from '../../../testing/createTestEditor';
+import replaceBackslashEscapes from '../replaceBackslashEscapes';
+import replaceBulletLists from '../replaceBulletLists';
+import replaceFormatCharacters from '../replaceFormatCharacters';
+import replaceInlineHtml from '../replaceInlineHtml';
+import replaceLinks from '../replaceLinks';
+import visibleEditorText from '../../../testing/visibleEditorText';
+import backspaceOnce from '../../../testing/backspaceOnce';
+
+jest.retryTimes(2);
+
+interface TestCase {
+	label: string;
+	markdown: string;
+	renderedText: string;
+	selectionFrom: number;
+	selectionTo: number;
+	expectedSyntaxTreeTags: string[];
+	extensions: Extension[];
+}
+
+const testCases: TestCase[] = [
+	{
+		label: 'bold text',
+		markdown: '**bold**\n',
+		renderedText: 'bold',
+		selectionFrom: 2,
+		selectionTo: 6,
+		expectedSyntaxTreeTags: ['StrongEmphasis'],
+		extensions: [replaceFormatCharacters],
+	},
+	{
+		label: 'bold italic text',
+		markdown: '***bold italic***\n',
+		renderedText: 'bold italic',
+		selectionFrom: 3,
+		selectionTo: 14,
+		expectedSyntaxTreeTags: ['Emphasis', 'StrongEmphasis'],
+		extensions: [replaceFormatCharacters],
+	},
+	{
+		label: 'heading',
+		markdown: '# Heading\n',
+		renderedText: 'Heading',
+		selectionFrom: 2,
+		selectionTo: 9,
+		expectedSyntaxTreeTags: ['ATXHeading1'],
+		extensions: [replaceFormatCharacters],
+	},
+	{
+		label: 'blockquote',
+		markdown: '> Blockquote\n',
+		renderedText: ' Blockquote',
+		selectionFrom: 2,
+		selectionTo: 12,
+		expectedSyntaxTreeTags: ['Blockquote'],
+		extensions: [replaceFormatCharacters],
+	},
+	{
+		label: 'link',
+		markdown: '[Joplin website](https://joplinapp.org)\n',
+		renderedText: 'Joplin website',
+		selectionFrom: 1,
+		selectionTo: 15,
+		expectedSyntaxTreeTags: ['Link'],
+		extensions: [replaceLinks],
+	},
+	{
+		label: 'link with bold text',
+		markdown: '[**Joplin website**](https://joplinapp.org)\n',
+		renderedText: 'Joplin website',
+		selectionFrom: 3,
+		selectionTo: 17,
+		expectedSyntaxTreeTags: ['Link', 'StrongEmphasis'],
+		extensions: [replaceFormatCharacters, replaceLinks],
+	},
+	{
+		label: 'inline HTML',
+		markdown: '<span style="color: red">red text</span>\n',
+		renderedText: 'red text',
+		selectionFrom: 25,
+		selectionTo: 33,
+		expectedSyntaxTreeTags: ['HTMLTag'],
+		extensions: [replaceInlineHtml],
+	},
+	{
+		label: 'escaped asterisks',
+		markdown: '\\*literal asterisk\\*\n',
+		renderedText: '*literal asterisk*',
+		selectionFrom: 1,
+		selectionTo: 20,
+		expectedSyntaxTreeTags: ['Escape'],
+		extensions: [replaceBackslashEscapes],
+	},
+	{
+		label: 'bullet list item',
+		markdown: '- item\n',
+		renderedText: '- item',
+		selectionFrom: 2,
+		selectionTo: 6,
+		expectedSyntaxTreeTags: ['BulletList'],
+		extensions: [replaceBulletLists],
+	},
+	{
+		label: 'bullet list item with selected leading space',
+		markdown: '- item\n',
+		renderedText: '- item',
+		selectionFrom: 1,
+		selectionTo: 6,
+		expectedSyntaxTreeTags: ['BulletList'],
+		extensions: [replaceBulletLists],
+	},
+];
+
+const directedTestCases = testCases.flatMap(testCase => [
+	{
+		...testCase,
+		direction: 'forward',
+		selection: EditorSelection.range(testCase.selectionFrom, testCase.selectionTo),
+	},
+	{
+		...testCase,
+		direction: 'backward',
+		selection: EditorSelection.range(testCase.selectionTo, testCase.selectionFrom),
+	},
+]);
+
+// Temporary CI diagnostic. Intermittently, `expect(textContent).toBe('red text')` fails with
+// Expected: "red text" / Received: "red text" - visually identical but unequal, so the actual
+// string differs by an invisible codepoint (e.g. char code 160/8203 where the healthy value has
+// a normal space, 32). This logs the char codes on mismatch to identify it. Remove once diagnosed.
+const expectTextContentToBe = (actual: string, expected: string) => {
+	if (actual !== expected) {
+		// eslint-disable-next-line no-console
+		console.error('FLAKE_DIAGNOSTIC', JSON.stringify({
+			expectedCodes: [...expected].map(character => character.charCodeAt(0)),
+			actualCodes: [...actual].map(character => character.charCodeAt(0)),
+		}));
+	}
+	expect(actual).toBe(expected);
+};
+
+describe('makeInlineReplaceExtension', () => {
+	beforeAll(() => {
+		jest.useFakeTimers({ advanceTimers: true });
+	});
+
+	it.each(directedTestCases)('should include Markdown syntax for $label in a $direction mouse selection', async ({
+		markdown, renderedText, selection, expectedSyntaxTreeTags, extensions,
+	}) => {
+		const editor = await createTestEditor(
+			markdown,
+			EditorSelection.cursor(markdown.length),
+			expectedSyntaxTreeTags,
+			extensions,
+		);
+
+		try {
+			editor.dom.dispatchEvent(new MouseEvent('mousedown', { button: 0 }));
+			editor.dispatch({ selection, userEvent: 'select.pointer' });
+			jest.runAllTimers();
+
+			const expectedSelection = {
+				anchor: selection.anchor,
+				head: selection.head,
+			};
+			expect(editor.state.selection.main).toMatchObject(expectedSelection);
+			expectTextContentToBe(visibleEditorText(editor), renderedText);
+
+			editor.dom.ownerDocument.dispatchEvent(new MouseEvent('mouseup', { button: 0 }));
+			await jest.runAllTimersAsync();
+
+			expect(editor.state.selection.main).toMatchObject(expectedSelection);
+			expectTextContentToBe(visibleEditorText(editor), markdown.trimEnd());
+		} finally {
+			editor.destroy();
+		}
+	});
+
+	it('should update markdown decorations when the mouse is down and the document changes', async () => {
+		const markdown = '**test**...';
+		const initialSelection = EditorSelection.cursor(markdown.length);
+		const editor = await createTestEditor(
+			'**test**...',
+			initialSelection,
+			['StrongEmphasis'],
+			[replaceFormatCharacters],
+		);
+
+		try {
+			editor.dom.dispatchEvent(new MouseEvent('mousedown', { button: 0 }));
+			editor.dispatch({ selection: initialSelection, userEvent: 'select.pointer' });
+			jest.runAllTimers();
+
+			expectTextContentToBe(visibleEditorText(editor), 'test...');
+
+			backspaceOnce(editor);
+			expectTextContentToBe(visibleEditorText(editor), 'test..');
+			backspaceOnce(editor);
+			expectTextContentToBe(visibleEditorText(editor), 'test.');
+			backspaceOnce(editor);
+			expectTextContentToBe(visibleEditorText(editor), '**test**');
+
+			editor.dom.ownerDocument.dispatchEvent(new MouseEvent('mouseup', { button: 0 }));
+			await jest.runAllTimersAsync();
+
+			expectTextContentToBe(visibleEditorText(editor), '**test**');
+		} finally {
+			editor.destroy();
+		}
+	});
+});

packages/editor/CodeMirror/extensions/rendering/utils/makeInlineReplaceExtension.ts

@@ -0,0 +1,185 @@
+// Ref: https://codemirror.net/examples/bundle/
+// and  https://codemirror.net/examples/decoration/
+
+import { EditorView, Decoration, DecorationSet, WidgetType } from '@codemirror/view';
+import { ViewPlugin, ViewUpdate } from '@codemirror/view';
+import { syntaxTree } from '@codemirror/language';
+import { EditorSelection, EditorState, Range, StateEffect } from '@codemirror/state';
+import { SyntaxNodeRef } from '@lezer/common';
+import { ReplacementExtension } from '../types';
+import nodeIntersectsSelection from './nodeIntersectsSelection';
+import clampSelectionToDocument from '../../../utils/clampSelectionToDocument';
+
+const updateInlineDecorationsEffect = StateEffect.define();
+
+interface MouseSelectionState {
+	initialSelection: EditorSelection;
+}
+
+interface VisibleRange {
+	from: number;
+	to: number;
+}
+
+export const makeInlineReplaceExtension = (extensionSpec: ReplacementExtension) => ViewPlugin.fromClass(class {
+	public decorations: DecorationSet = Decoration.set([]);
+	private mouseSelectionBefore_: MouseSelectionState|null = null;
+
+	public constructor(private view: EditorView) {
+		view.dom.addEventListener('mousedown', this.onMouseDown, true);
+		view.dom.ownerDocument.addEventListener('mouseup', this.onMouseUp);
+		this.updateDecorations(view.state, view.visibleRanges);
+	}
+
+	public destroy() {
+		this.view.dom.removeEventListener('mousedown', this.onMouseDown, true);
+		this.view.dom.ownerDocument.removeEventListener('mouseup', this.onMouseUp);
+	}
+
+	private onMouseDown = (event: MouseEvent) => {
+		if (event.button === 0) {
+			this.mouseSelectionBefore_ = { initialSelection: this.view.state.selection };
+		}
+	};
+
+	private onMouseUp = () => {
+		if (this.mouseSelectionBefore_) {
+			// To prevent unnecessary scroll on iOS, decoration changes need to
+			// happen *after* the gesture ends.
+			requestAnimationFrame(() => {
+				this.mouseSelectionBefore_ = null;
+				this.view.dispatch({
+					effects: updateInlineDecorationsEffect.of(null),
+				});
+			});
+		}
+	};
+
+	private updateDecorations(state: EditorState, visibleRanges: readonly VisibleRange[]) {
+		const doc = state.doc;
+		let selection = state.selection;
+		if (this.mouseSelectionBefore_?.initialSelection) {
+			selection = clampSelectionToDocument(this.mouseSelectionBefore_.initialSelection, doc);
+		}
+		if (this.mouseSelectionBefore_) {
+			state = state.update({ selection }).state;
+		}
+		const cursorLine = doc.lineAt(selection.main.anchor);
+
+		const parentTagCounts = new Map<string, number>();
+		const decorateNode = (node: SyntaxNodeRef) => {
+			const widgetOrDecoration = extensionSpec.createDecoration(node, state, parentTagCounts);
+			let decoration;
+			if (widgetOrDecoration instanceof WidgetType) {
+				decoration = Decoration.replace({
+					widget: widgetOrDecoration,
+				});
+			} else if (widgetOrDecoration instanceof Decoration) {
+				decoration = widgetOrDecoration;
+			}
+
+			if (decoration) {
+				const range = extensionSpec.getDecorationRange?.(node, state, parentTagCounts) ?? [node.from, node.to];
+				const rangeLineFrom = doc.lineAt(range[0]);
+				const rangeLineTo = range.length === 2 ? doc.lineAt(range[1]) : rangeLineFrom;
+
+				// A different start/end line causes errors.
+				if (rangeLineFrom.number === rangeLineTo.number) {
+					if (range.length === 1) {
+						widgets.push(decoration.range(range[0]));
+					} else {
+						widgets.push(decoration.range(range[0], range[1]));
+					}
+				}
+			}
+		};
+
+		let widgets: Range<Decoration>[] = [];
+		for (const { from, to } of visibleRanges) {
+			parentTagCounts.clear();
+			syntaxTree(state).iterate({
+				from, to,
+				enter: node => {
+					parentTagCounts.set(node.name, (parentTagCounts.get(node.name) ?? 0) + 1);
+
+					const strategy = extensionSpec.getRevealStrategy?.(node, state, parentTagCounts) ?? 'line';
+
+					let isSelected = false;
+					if (typeof strategy === 'boolean') {
+						isSelected = strategy;
+					} else if (strategy === 'line') {
+						const nodeLine = doc.lineAt(node.from);
+						const lineContainsSelection = cursorLine.number === nodeLine.number;
+						isSelected = lineContainsSelection || nodeIntersectsSelection(selection, node);
+					} else if (strategy === 'select') {
+						isSelected = nodeIntersectsSelection(selection, node);
+					} else if (strategy === 'active') {
+						const parent = node.node.parent;
+						isSelected = nodeIntersectsSelection(selection, node) || (!!parent && nodeIntersectsSelection(selection, parent));
+					}
+
+					const shouldHide = (
+						(extensionSpec.hideWhenContainsSelection ?? true) && isSelected
+					);
+
+					if (!shouldHide) {
+						decorateNode(node);
+					}
+				},
+				leave: node => {
+					parentTagCounts.set(node.name, (parentTagCounts.get(node.name) ?? 0) - 1);
+				},
+			});
+		}
+		this.decorations = Decoration.set(widgets, true);
+
+		if (extensionSpec.mergeNeighbors && widgets.length > 0) {
+			const originalLength = widgets.length;
+			widgets = [];
+
+			const iter = this.decorations.iter();
+			let previous = iter.value;
+			let previousFrom = iter.from;
+			let previousTo = iter.to;
+			widgets.push(iter.value.range(iter.from, iter.to));
+
+			for (iter.next(); iter.value; iter.next()) {
+				let from = iter.from;
+				if (previousTo === iter.from && previous.eq(iter.value)) {
+					from = previousFrom;
+					widgets.pop();
+				}
+				widgets.push(iter.value.range(from, iter.to));
+
+				previous = iter.value;
+				previousTo = iter.to;
+				previousFrom = from;
+			}
+
+			if (widgets.length < originalLength) {
+				this.decorations = Decoration.set(widgets, true);
+			}
+		}
+	}
+
+	public update(update: ViewUpdate) {
+		const forceUpdate = update.transactions.some(transaction => (
+			transaction.effects.some(effect => effect.is(updateInlineDecorationsEffect))
+			|| extensionSpec.shouldFullReRender?.(transaction)
+		));
+
+		// Document changes move the selection, so the original selection may no longer
+		// be valid:
+		if (update.docChanged) {
+			this.mouseSelectionBefore_ = null;
+		}
+
+		if (update.docChanged || update.viewportChanged || update.selectionSet || forceUpdate) {
+			this.updateDecorations(update.state, update.view.visibleRanges);
+		}
+	}
+}, {
+	decorations: view => view.decorations,
+});
+
+export default makeInlineReplaceExtension;

packages/editor/CodeMirror/extensions/rendering/utils/nodeIntersectsSelection.ts

@@ -0,0 +1,17 @@
+import { EditorSelection } from '@codemirror/state';
+import { SyntaxNodeRef } from '@lezer/common';
+
+const nodeIntersectsSelection = (selection: EditorSelection, node: SyntaxNodeRef) => {
+	const mainSelection = selection.main;
+
+	const nodeContains = (point: number) => {
+		return point >= node.from && point <= node.to;
+	};
+	const selectionContains = (point: number) => {
+		return point >= mainSelection.from && point <= mainSelection.to;
+	};
+	return nodeContains(mainSelection.from) || nodeContains(mainSelection.to)
+		|| selectionContains(node.from) || selectionContains(node.to);
+};
+
+export default nodeIntersectsSelection;