feat: Initial clean release of CHM

packages/tools/fuzzer/ipc/ClientPool.ts

@@ -0,0 +1,142 @@
+import Logger from '@joplin/utils/Logger';
+import ActionTracker from '../model/ActionTracker';
+import Client from './Client';
+import { FuzzContext } from '../types';
+import { join } from 'path';
+import { mkdir } from 'fs-extra';
+import { readdir } from 'fs/promises';
+
+type ClientFilter = (client: Client)=> boolean;
+
+const logger = Logger.create('ClientPool');
+
+export default class ClientPool {
+	public static async create(context: FuzzContext) {
+		return new ClientPool(context);
+	}
+
+	public static async fromSnapshot(snapshotDirectory: string, actionTracker: ActionTracker, context: FuzzContext) {
+		const pool = await ClientPool.create(context);
+
+		const matchingDirectories = (await readdir(snapshotDirectory))
+			.filter(child => child.startsWith('client-'))
+			.map(child => join(snapshotDirectory, child))
+			.sort();
+
+		const accounts = new Map();
+		for (const clientDirectory of matchingDirectories) {
+			const client = await Client.fromSnapshotDirectory(clientDirectory, actionTracker, context, accounts);
+			pool.clients_.push(client);
+			pool.listenForClientClose_(client);
+		}
+
+		return pool;
+	}
+
+	private clients_: Client[] = [];
+
+	private constructor(
+		private readonly context_: FuzzContext,
+	) {
+	}
+
+	private listenForClientClose_(client: Client) {
+		client.onClose(() => {
+			this.clients_ = this.clients_.filter(other => other !== client);
+		});
+	}
+
+	public async saveSnapshot(outputDirectory: string) {
+		let i = 0;
+		for (const client of this.clients) {
+			const outputChildDirectory = join(outputDirectory, `client-${i++}`);
+			await mkdir(outputChildDirectory);
+			await client.saveSnapshot(outputChildDirectory);
+		}
+	}
+
+	public async newClient(model: ActionTracker) {
+		const client = await Client.create(model, this.context_);
+
+		this.clients_.push(client);
+		this.listenForClientClose_(client);
+	}
+
+	public async createRandomInitialItemsAndSync() {
+		for (const client of this.clients) {
+			logger.info('Creating items for ', client.email);
+			const actionCount = this.context_.randomFrom([0, 10, 100]);
+			await client.createOrUpdateMany(actionCount, { createProbability: 1, updateProbability: 1, deleteProbability: 1 });
+
+			await client.sync();
+		}
+	}
+
+	public clientsByEmail(email: string) {
+		return this.clients.filter(client => client.email === email);
+	}
+
+	public clientById(id: number) {
+		const client = this.clients[id];
+		if (!client) throw new Error(`Not found: ${id}`);
+		return client;
+	}
+
+	public getClientId(client: Client): number {
+		const index = this.clients.indexOf(client);
+		if (index === -1) throw new Error(`Not found: ${client}`);
+		return index;
+	}
+
+	public randomClient(filter: ClientFilter = ()=>true) {
+		const clients = this.clients_.filter(filter);
+		return clients[
+			this.context_.randInt(0, clients.length)
+		];
+	}
+
+	public async newWithSameAccount(sourceClient: Client) {
+		const client = await sourceClient.createClientOnSameAccount();
+		this.listenForClientClose_(client);
+		this.clients_ = [...this.clients_, client];
+		return client;
+	}
+
+	public othersWithSameAccount(client: Client) {
+		return this.clients_.filter(other => other !== client && other.hasSameAccount(client));
+	}
+
+	public async checkState() {
+		for (const client of this.clients_) {
+			await client.checkState();
+		}
+	}
+
+	public async syncAll() {
+		// Sync all clients at roughly the same time. Some sync bugs are only apparent
+		// when multiple clients are syncing simultaneously.
+		await Promise.all(this.clients_.map(c => c.sync()));
+
+		// Note: For more deterministic behavior, sync clients individually instead:
+		//   for (const client of this.clients_) { await client.sync(); }
+	}
+
+	public get clients() {
+		return [...this.clients_];
+	}
+
+	public helpText() {
+		return this.clients_.map(client => client.getHelpText()).join('\n\n');
+	}
+
+	public async close() {
+		for (const client of this.clients) {
+			try {
+				await client.close();
+			} catch (error) {
+				logger.warn('Failed to close client', error);
+			}
+		}
+	}
+}
+

packages/tools/fuzzer/ipc/Server.ts

@@ -0,0 +1,162 @@
+import { join, resolve } from 'path';
+import { HttpMethod, Json, UserData } from '../types';
+import JoplinServerApi from '@joplin/lib/JoplinServerApi';
+import { Env } from '@joplin/lib/models/Setting';
+import execa = require('execa');
+import { msleep } from '@joplin/utils/time';
+import Logger from '@joplin/utils/Logger';
+import { strict as assert } from 'assert';
+import { copy, exists } from 'fs-extra';
+import { copyFile } from 'fs/promises';
+
+const logger = Logger.create('Server');
+
+const createApi = async (serverUrl: string, adminAuth: UserData) => {
+	const api = new JoplinServerApi({
+		baseUrl: () => serverUrl,
+		userContentBaseUrl: () => serverUrl,
+		password: () => adminAuth.password,
+		username: () => adminAuth.email,
+		session: ()=>null,
+		apiKey: ()=>'',
+		env: Env.Dev,
+	});
+	await api.loadSession();
+	return api;
+};
+
+interface ServerConfig {
+	baseUrl: string;
+	baseDirectory: string;
+	useRunningServer: boolean;
+	adminAuth: UserData;
+}
+
+interface FromSnapshotOptions extends ServerConfig {
+	snapshotDirectory: string;
+}
+
+export default class Server {
+	private api_: JoplinServerApi|null = null;
+	private serverUrl_: string;
+	private adminAuth_: UserData;
+	private server_: execa.ExecaChildProcess<string>|null;
+	private baseDirectory_: string;
+
+	public constructor(config: ServerConfig) {
+		this.adminAuth_ = config.adminAuth;
+
+		const serverDir = resolve(config.baseDirectory);
+		this.baseDirectory_ = serverDir;
+
+		this.serverUrl_ = config.baseUrl;
+
+		// Code in other places (e.g. the JoplinServerApi) assumes that serverUrl_ ends with a "/"
+		if (!this.serverUrl_.endsWith('/')) {
+			this.serverUrl_ = `${this.serverUrl_}/`;
+		}
+
+		const mainEntrypoint = join(serverDir, 'dist', 'index.js');
+		if (!config.useRunningServer) {
+			this.server_ = execa.node(mainEntrypoint, [
+				'--env', 'dev',
+			], {
+				env: {
+					JOPLIN_IS_TESTING: '1',
+				},
+				cwd: serverDir,
+				stdin: 'ignore', // No stdin
+				// For debugging:
+				stderr: process.stderr,
+				// stdout: process.stdout,
+			});
+		} else {
+			this.server_ = null;
+		}
+	}
+
+	private static assertCanUseSnapshots_() {
+		if (process.env.SQLITE_DATABASE) {
+			throw new Error(`Unsupported: Creating snapshots of a non-default database (${JSON.stringify(process.env.SQLITE_DATABASE)}) is not supported. Skipping...`);
+		}
+		if ((process.env.DB_CLIENT ?? 'sqlite') !== 'sqlite') {
+			throw new Error(`Not supported: Creating snapshots of a non-sqlite database is not supported (DB_CLIENT: ${process.env.DB_CLIENT}). Skipping...`);
+		}
+	}
+
+	public assertCanUseSnapshots() {
+		Server.assertCanUseSnapshots_();
+
+		if (!this.server_) {
+			throw new Error('Not supported: Creating snapshots using an external server instance.');
+		}
+	}
+
+	public static async fromSnapshot({
+		baseDirectory: serverBaseDirectory, snapshotDirectory, ...config
+	}: FromSnapshotOptions) {
+		this.assertCanUseSnapshots_();
+
+		const serverDatabaseFile = join(serverBaseDirectory, 'db-dev.sqlite');
+		logger.info('Restoring', serverDatabaseFile, '... Replacing with version from snapshot...');
+		await copy(join(snapshotDirectory, 'server', 'db-dev.sqlite'), serverDatabaseFile);
+
+		return new Server({
+			baseDirectory: serverBaseDirectory,
+			...config,
+		});
+	}
+
+	public async saveSnapshot(outputDirectory: string) {
+		Server.assertCanUseSnapshots_();
+
+		// Note: Assumes that the server is using SQLite!
+		const databasePath = join(this.baseDirectory_, 'db-dev.sqlite');
+		logger.info('Creating snapshot of', databasePath, '...');
+
+		assert.ok(await exists(outputDirectory));
+		const destination = join(outputDirectory, 'db-dev.sqlite');
+		await copyFile(databasePath, destination);
+	}
+
+	public get url() {
+		return this.serverUrl_;
+	}
+
+	public async checkConnection() {
+		let lastError;
+		for (let retry = 0; retry < 30; retry++) {
+			try {
+				const response = await fetch(`${this.serverUrl_}api/ping`);
+				if (response.ok) {
+					return true;
+				}
+			} catch (error) {
+				lastError = error;
+			}
+			await msleep(500);
+		}
+		if (lastError) {
+			throw lastError;
+		}
+		return false;
+	}
+
+	public async execApi(method: HttpMethod, route: string, action: Json|undefined): Promise<Json> {
+		this.api_ ??= await createApi(this.serverUrl_, this.adminAuth_);
+		logger.debug('API EXEC', method, route, action);
+		const result = await this.api_.exec(method, route, {}, action);
+		return result;
+	}
+
+	public async close() {
+		if (this.server_) {
+			this.server_.cancel();
+			logger.info('Closed the server.');
+		} else {
+			logger.info('Server not closed: Running using an external server.');
+		}
+	}
+}
+
+