feat: Initial clean release of CHM

packages/tools/fuzzer/utils/ProgressBar.ts

@@ -0,0 +1,28 @@
+import { stdout } from 'process';
+
+export default class ProgressBar {
+	private isFirst_ = true;
+	private lastLength_ = 0;
+	public constructor(private prefix_: string) {}
+	public update(countDone: number, total: number) {
+		if (!stdout.isTTY) return;
+
+		if (this.isFirst_) {
+			this.isFirst_ = false;
+		}
+
+		const percent = Math.round(countDone / total * 100);
+		const message = `\r${this.prefix_}: ${percent}% (${countDone}/${total})`;
+		stdout.write(message.padEnd(this.lastLength_));
+
+		this.lastLength_ = message.length;
+	}
+
+	public complete() {
+		if (!this.isFirst_) {
+			stdout.write(`\r${this.prefix_}: Done`.padEnd(this.lastLength_));
+			stdout.write('\n');
+		}
+	}
+}
+

packages/tools/fuzzer/utils/SeededRandom.ts

@@ -0,0 +1,70 @@
+
+// SeededRandom provides a very simple random number generator
+// that can be seeded (since NodeJS built-ins can't).
+//
+// See also:
+// - https://arxiv.org/pdf/1704.00358
+// - https://en.wikipedia.org/wiki/Middle-square_method
+
+// Some large odd number, see https://en.wikipedia.org/wiki/Weyl_sequence
+const step = BigInt('0x12345678ABCDE123'); // uint64
+const maxSize = BigInt(1) << BigInt(64);
+
+const extractMiddle = (value: bigint, halfSize: bigint) => {
+	// Remove the lower quarter
+	const quarterSize = halfSize / BigInt(2);
+	value >>= quarterSize;
+
+	// Remove the upper quarter
+	const halfMaximumValue = BigInt(1) << halfSize;
+	value %= halfMaximumValue;
+
+	return value;
+};
+
+export interface RandomState {
+	value: bigint|string;
+	step: bigint|string;
+}
+
+export default class SeededRandom {
+	private value_: bigint;
+	private nextStep_: bigint = step;
+	private halfSize_ = BigInt(32);
+
+	public constructor(
+		seed: number|bigint|string,
+	) {
+		this.value_ = typeof seed !== 'bigint' ? BigInt(seed) : seed;
+	}
+
+	public static fromState({ value, step }: RandomState) {
+		const random = new SeededRandom(value);
+		random.nextStep_ = typeof step !== 'bigint' ? BigInt(step) : step;
+		return random;
+	}
+
+	// Returns a state that can be used to save/restore this random number generator.
+	public get state(): RandomState {
+		return { value: this.value_, step: this.nextStep_ };
+	}
+
+	public next() {
+		this.value_ = this.value_ * this.value_ + this.nextStep_;
+
+		// Move to the next item in the sequence. Mod to prevent from getting
+		// too large. See https://en.wikipedia.org/wiki/Weyl_sequence.
+		this.nextStep_ = (step + this.nextStep_) % maxSize;
+
+		this.value_ = extractMiddle(this.value_, this.halfSize_);
+		return this.value_;
+	}
+
+	// The resultant range includes `a` but excludes `b`.
+	public nextInRange(a: number, b: number) {
+		if (b <= a + 1) return a;
+
+		const range = b - a;
+		return Number(this.next() % BigInt(range)) + a;
+	}
+}

packages/tools/fuzzer/utils/diffSortedStringArrays.test.ts

@@ -0,0 +1,34 @@
+import diffSortedStringArrays from './diffSortedStringArrays';
+
+describe('diffSortedStringArrays', () => {
+	test.each([
+		[
+			['a'],
+			[],
+			{ unexpected: ['a'], missing: [] },
+		],
+		[
+			['a', 'b'],
+			['a', 'b'],
+			{ unexpected: [], missing: [] },
+		],
+		[
+			[],
+			['a'],
+			{ unexpected: [], missing: ['a'] },
+		],
+		[
+			['a'],
+			['b', 'c'],
+			{ unexpected: ['a'], missing: ['b', 'c'] },
+		],
+		[
+			['a', 'b', 'c', 'd', 'e', 'f'],
+			['a', 'e'],
+			{ unexpected: ['b', 'c', 'd', 'f'], missing: [] },
+		],
+	])('should diff string arrays: %j, %j', (actual, expected, diff) => {
+		expect(diffSortedStringArrays(actual, expected)).toEqual(diff);
+	});
+});
+

packages/tools/fuzzer/utils/diffSortedStringArrays.ts

@@ -0,0 +1,58 @@
+
+// Input arrays must be sorted
+const diffSortedStringArrays = (actual: string[], expected: string[]) => {
+	const missing = [];
+	const unexpected = [];
+
+	let indexActual = 0;
+	let indexExpected = 0;
+	for (;
+		indexActual < actual.length && indexExpected < expected.length;
+		indexActual++, indexExpected++
+	) {
+
+		const itemActual = actual[indexActual];
+		const itemExpected = expected[indexExpected];
+
+		if (itemActual !== itemExpected) {
+			let found = false;
+			// Case 1: The expected item is present eventually, after a few unexpected item:
+			for (let i = indexActual + 1; i < actual.length; i++) {
+				if (actual[i] === itemExpected) {
+					// Everything before the found item shouldn't have been present:
+					unexpected.push(actual.slice(indexActual, i));
+
+					indexActual = i;
+					found = true;
+					break;
+				}
+			}
+
+			// Case 2: The expected item wasn't present at all:
+			if (!found) {
+				missing.push(itemExpected);
+
+				// Revisit the current item in "actual" on the next loop:
+				indexActual --;
+			}
+		}
+	}
+
+	// Handle any unexpected items at the end
+	if (indexActual < actual.length) {
+		unexpected.push(actual.slice(indexActual));
+	}
+	if (indexExpected < expected.length) {
+		missing.push(expected.slice(indexExpected));
+	}
+
+	return {
+		// Items that were present in the actual state, but are missing from the expected state
+		unexpected: unexpected.flat(),
+		// Items that were present in the expected state, but missing from the actual state.
+		missing: missing.flat(),
+	};
+};
+
+export default diffSortedStringArrays;
+

packages/tools/fuzzer/utils/extractResourceIds.ts

@@ -0,0 +1,7 @@
+import { extractResourceUrls } from '@joplin/lib/urlUtils';
+
+const extractResourceIds = (text: string) => {
+	return extractResourceUrls(text).map(item => item.itemId);
+};
+
+export default extractResourceIds;

packages/tools/fuzzer/utils/getBinaryDiffDebugMessage.ts

@@ -0,0 +1,77 @@
+
+// Provides additional debugging information for differing strings
+const getDiffDebugMessage = (actual: string, expected: string) => {
+	if (actual === expected) return '';
+
+	const diffMessage = [];
+
+	// List all characters present in actual that were not present in expected.
+	// This helps debug issues in which the unicode replacement character has been
+	// added incorrectly:
+	{
+		const inExpected = new Set();
+		for (let i = 0; i < expected.length; i++) {
+			inExpected.add(expected.charAt(i));
+		}
+
+		const unexpected = new Set();
+		for (let i = 0; i < actual.length; i++) {
+			const char = actual.charAt(i);
+			if (!inExpected.has(char)) {
+				unexpected.add(actual.charAt(i));
+			}
+		}
+
+		if (unexpected.size) {
+			diffMessage.push('Characters found in actual that were not in the expected state:\n');
+			diffMessage.push('  ', JSON.stringify([...unexpected]), '\n\n');
+		}
+	}
+
+	const actualBinary = new Uint8Array(new TextEncoder().encode(actual));
+	const expectedBinary = new Uint8Array(new TextEncoder().encode(expected));
+
+	if (actualBinary.length > expectedBinary.length) {
+		diffMessage.push(`Actual is longer than expected by ${actualBinary.length - expectedBinary.length}\n`);
+	} else if (expectedBinary.length > actualBinary.length) {
+		diffMessage.push(`Expected is longer than actual by ${expectedBinary.length - actualBinary.length}\n`);
+	}
+
+	for (let i = 0; i < actualBinary.length; i++) {
+		if (i >= expectedBinary.length) {
+			break;
+		}
+
+		if (expectedBinary[i] !== actualBinary[i]) {
+			diffMessage.push(
+				'First binary difference at position', i, `(0x${i.toString(16)})`, ': ', expectedBinary[i], '!=', actualBinary[i], '(expected != actual)',
+				'\n\tContext: expected[i-3:i+5] = ', [...expectedBinary.slice(Math.max(i - 3, 0), i + 5)],
+				'\n\tContext: actual[i-3 : i+5] = ', [...actualBinary.slice(Math.max(i - 3, 0), i + 5)],
+				'\n\tactual.byteLength = ', actualBinary.length, ', expected.byteLength = ', expectedBinary.length,
+				'\n\n',
+			);
+			break;
+		}
+	}
+
+	// Also log information about the last difference
+	for (let i = 0; i < Math.min(actualBinary.length, expectedBinary.length); i++) {
+		const indexExpected = expectedBinary.length - i - 1;
+		const indexActual = actualBinary.length - i - 1;
+		if (expectedBinary[indexExpected] !== actualBinary[indexActual]) {
+			diffMessage.push(
+				'Last binary difference (working from end)', ': ',
+				expectedBinary[indexExpected], '!=', actualBinary[indexActual], `(expected[${indexExpected}] != actual[${indexActual}])`,
+				'\n\tContext: expected[a-6:a+3] = ', [...expectedBinary.slice(Math.max(indexExpected - 6, 0), indexExpected + 3)],
+				'\n\tContext: actual[b-6 : b+3] = ', [...actualBinary.slice(Math.max(indexActual - 6, 0), indexActual + 3)],
+				'\n\twhere expected.byteLength = ', expectedBinary.byteLength, 'and actual.byteLength = ', actualBinary.byteLength,
+				'\n\n',
+			);
+			break;
+		}
+	}
+
+	return diffMessage.join(' ');
+};
+
+export default getDiffDebugMessage;

packages/tools/fuzzer/utils/getNumberProperty.ts

@@ -0,0 +1,11 @@
+import getProperty from './getProperty';
+
+const getNumberProperty = (object: unknown, propertyName: string) => {
+	const value = getProperty(object, propertyName);
+	if (typeof value !== 'number') {
+		throw new Error(`Property value is not a string (is ${typeof value})`);
+	}
+	return value;
+};
+
+export default getNumberProperty;

packages/tools/fuzzer/utils/getProperty.ts

@@ -0,0 +1,17 @@
+
+const getProperty = (object: unknown, propertyName: string) => {
+	if (typeof object !== 'object' || object === null) {
+		throw new Error(`Cannot access property ${JSON.stringify(propertyName)} on non-object`);
+	}
+
+	if (!(propertyName in object)) {
+		throw new Error(
+			`No such property ${JSON.stringify(propertyName)} in object. Available keys: (${JSON.stringify(Object.keys(object))})`,
+		);
+	}
+
+	return object[propertyName as keyof object];
+};
+
+export default getProperty;
+

packages/tools/fuzzer/utils/getStringProperty.ts

@@ -0,0 +1,11 @@
+import getProperty from './getProperty';
+
+const getStringProperty = (object: unknown, propertyName: string) => {
+	const value = getProperty(object, propertyName);
+	if (typeof value !== 'string') {
+		throw new Error(`Property value is not a string (is ${typeof value})`);
+	}
+	return value;
+};
+
+export default getStringProperty;

packages/tools/fuzzer/utils/hangingIndent.ts

@@ -0,0 +1,7 @@
+
+// Hanging indent: Indents all lines after the first
+const hangingIndent = (text: string, indentation = '   ') => {
+	return text.replace(/\n/g, `\n${indentation}`);
+};
+
+export default hangingIndent;

packages/tools/fuzzer/utils/openDebugSession.ts

@@ -0,0 +1,50 @@
+import Logger from '@joplin/utils/Logger';
+import ClientPool from '../ipc/ClientPool';
+import { createInterface } from 'readline/promises';
+
+const logger = Logger.create('openDebugSession');
+
+const openDebugSession = async (clients: ClientPool) => {
+	const allClients = clients.clients;
+	const clientChoices = allClients.map((client, index) => {
+		return `${index}: ${client.label}`;
+	}).join('\n');
+
+	const askForClient = async (questionPrefix = '') => {
+		// Recreate the readline interface each time to avoid conflicting
+		// with the debug sessions for the individual clients.
+		const readlineInterface = createInterface({
+			input: process.stdin,
+			output: process.stdout,
+		});
+		try {
+			const clientChoice = await readlineInterface.question(
+				`${questionPrefix}Select a client from:\n${clientChoices}\nclient: `,
+			);
+			if (clientChoice.trim() === '' || clientChoice === 'exit') {
+				return null;
+			}
+
+			const asNumber = Number(clientChoice);
+			if (!isFinite(asNumber) || Math.floor(asNumber) !== asNumber) {
+				return askForClient('Please input an integer. ');
+			}
+
+			if (asNumber < 0 || asNumber >= allClients.length) {
+				return askForClient('Choice out of range. ');
+			}
+
+			return allClients[asNumber];
+		} finally {
+			readlineInterface.close();
+		}
+	};
+
+	for (let client = await askForClient(); client; client = await askForClient()) {
+		logger.info('Switching to client', client.getHelpText());
+
+		await client.startCliDebugSession();
+	}
+};
+
+export default openDebugSession;

packages/tools/fuzzer/utils/randomId.test.ts

@@ -0,0 +1,12 @@
+import randomId from './randomId';
+
+describe('randomId', () => {
+	test('should generate a 32-character alphanumeric ID', () => {
+		expect(
+			randomId((_low, high) => high - 1)(),
+		).toBe('ffffffffffffffffffffffffffffffff');
+		expect(
+			randomId((low, _high) => low)(),
+		).toBe('00000000000000000000000000000000');
+	});
+});

packages/tools/fuzzer/utils/randomId.ts

@@ -0,0 +1,16 @@
+
+type OnNextRandom = (lowInclusive: number, highExclusive: number)=> number;
+
+const randomId = (nextRandomInteger: OnNextRandom)=> () => {
+	const bytes = [];
+	for (let i = 0; i < 16; i++) {
+		bytes.push(nextRandomInteger(0, 256));
+	}
+
+	return Buffer.from(bytes)
+		.toString('hex')
+		.toLowerCase()
+		.padStart(32, '0');
+};
+
+export default randomId;

packages/tools/fuzzer/utils/randomString.ts

@@ -0,0 +1,31 @@
+import removeInvalidUtf8 from './removeInvalidUtf8';
+
+type OnRandomInt = (low: number, high: number)=> number;
+
+const randomString = (nextRandomInteger: OnRandomInt) => (length: number) => {
+	const charCodes = [];
+	for (let i = 0; i < length; i++) {
+		charCodes.push(nextRandomInteger(0, 0xFFFF));
+	}
+
+	let text = String.fromCharCode(...charCodes);
+	// Normalize to avoid differences when reading/writing content from Joplin clients.
+	// TODO: Can the note comparison logic be adjusted to remove some of this?
+	text = text.normalize();
+	text = text.replace(/[\r\b\f\v\0\u007F]/g, '!');
+	text = text.replace(/\p{C}/ug, '-'); // Other control characters
+
+	// Remove invalid UTF-8
+	text = removeInvalidUtf8(text);
+
+	// Attempt to work around issues related to unexpected differences in items when reading from
+	// the Joplin client: Remove certain invalid unicode:
+	if ('toWellFormed' in String.prototype && typeof String.prototype.toWellFormed === 'function') {
+		// toWellFormed requires Node >= v20
+		text = String.prototype.toWellFormed.call(text);
+	}
+
+	return text;
+};
+
+export default randomString;

packages/tools/fuzzer/utils/removeInvalidUtf8.ts

@@ -0,0 +1,3 @@
+
+const removeInvalidUtf8 = (text: string) => new TextDecoder().decode(new TextEncoder().encode(text));
+export default removeInvalidUtf8;

packages/tools/fuzzer/utils/retryWithCount.ts

@@ -0,0 +1,40 @@
+import Logger from '@joplin/utils/Logger';
+import { msleep } from '@joplin/utils/time';
+
+const logger = Logger.create('retryWithCount');
+
+interface FailureEvent {
+	error: Error;
+	willRetry: boolean;
+}
+
+interface Options {
+	count: number;
+	delayOnFailure?: (retryCount: number)=> number;
+	onFail: (event: FailureEvent)=> void|Promise<void>;
+}
+
+const retryWithCount = async (task: ()=> Promise<void>, { count, delayOnFailure, onFail }: Options) => {
+	let lastError: Error|null = null;
+	for (let retry = 0; retry < count; retry ++) {
+		try {
+			return await task();
+		} catch (error) {
+			lastError = error;
+
+			const willRetry = retry + 1 < count;
+			await onFail({ error, willRetry });
+
+			const delay = willRetry && delayOnFailure ? delayOnFailure(retry + 1) : 0;
+			if (delay) {
+				logger.info(`Retrying after ${delay}ms...`);
+				await msleep(delay);
+			}
+		}
+	}
+
+	if (lastError) throw lastError;
+};
+
+export default retryWithCount;
+