feat: Initial clean release of CHM

packages/tools/gulp/tasks/buildScriptIndexes.js

@@ -0,0 +1,83 @@
+// Takes a directory of TypeScript files and generate an index from it.
+
+const utils = require('../utils');
+const glob = require('glob');
+const rootDir = utils.rootDir();
+
+async function processDirectory(dir, indexFilePath = null, typeScriptType = null, imports = null, importNameTemplate = null, exportNameTemplate = null) {
+	if (!indexFilePath) indexFilePath = `${dir}/index.ts`;
+	if (!typeScriptType) typeScriptType = 'any';
+	if (!importNameTemplate) importNameTemplate = '* as FILE_NAME';
+	if (!exportNameTemplate) exportNameTemplate = 'FILE_NAME';
+
+	const tsFiles = glob.sync('{*.ts,*.tsx}', {
+		cwd: dir,
+	}).filter(f => `${dir}/${f}` !== indexFilePath)
+	//
+	// Exclude Jest test files to
+	// not include them in index.ts
+	//
+		.filter(f => !f.endsWith('.test.ts'));
+
+	tsFiles.sort();
+
+	const fileContent = [];
+
+	for (const tsFile of tsFiles) {
+		const f = utils.getFilename(tsFile);
+		fileContent.push(`import ${importNameTemplate.replace(/FILE_NAME/g, f)} from './${f}';`);
+	}
+
+	fileContent.push('');
+
+	if (imports) {
+		fileContent.push(imports);
+		fileContent.push('');
+	}
+
+	fileContent.push(`const index: ${typeScriptType}[] = [`);
+
+	for (const tsFile of tsFiles) {
+		const f = utils.getFilename(tsFile);
+		fileContent.push(`\t${exportNameTemplate.replace(/FILE_NAME/g, f)},`);
+	}
+
+	fileContent.push('];');
+
+	fileContent.push('');
+
+	fileContent.push('export default index;');
+
+	console.info(`Generating ${indexFilePath}...`);
+
+	await utils.insertContentIntoFile(
+		indexFilePath,
+		'// AUTO-GENERATED using `gulp buildScriptIndexes`',
+		fileContent.join('\n'),
+		true,
+	);
+}
+
+module.exports = {
+	src: '',
+	fn: async function() {
+		await processDirectory(`${rootDir}/packages/app-desktop/commands`);
+		await processDirectory(`${rootDir}/packages/app-desktop/gui/WindowCommandsAndDialogs/commands`);
+		await processDirectory(`${rootDir}/packages/app-desktop/gui/NoteEditor/commands`);
+		await processDirectory(`${rootDir}/packages/app-desktop/gui/NoteList/commands`);
+		await processDirectory(`${rootDir}/packages/app-desktop/gui/NoteListControls/commands`);
+		await processDirectory(`${rootDir}/packages/app-desktop/gui/Sidebar/commands`);
+		await processDirectory(`${rootDir}/packages/app-mobile/commands`);
+		await processDirectory(`${rootDir}/packages/app-mobile/components/screens/Note/commands`);
+		await processDirectory(`${rootDir}/packages/lib/commands`);
+
+		await processDirectory(
+			`${rootDir}/packages/lib/services/database/migrations`,
+			null,
+			'Migration',
+			'import { Migration } from \'../types\';',
+			'migrationFILE_NAME',
+			'migrationFILE_NAME',
+		);
+	},
+};

packages/tools/gulp/tasks/buildScriptIndexesRun.js

@@ -0,0 +1,12 @@
+// Allow running that task "buildScriptIndexes" without gulp
+
+const task = require('./buildScriptIndexes.js');
+
+const main = async () => {
+	await task.fn();
+};
+
+main().catch((error) => {
+	console.error(error);
+	process.exit(1);
+});

packages/tools/gulp/tasks/compileExtensions.js

@@ -0,0 +1,55 @@
+const fs = require('fs-extra');
+const utils = require('../utils');
+
+
+async function getSourceCode(dest) {
+	await utils.execCommand(`curl -o ${dest}/sqlite.tar.gz "https://www.sqlite.org/src/tarball/sqlite.tar.gz?r=release"`);
+	await utils.execCommand(`curl -o ${dest}/amalgamation.tar.gz "https://www.sqlite.org/2020/sqlite-autoconf-3330000.tar.gz"`);
+	await utils.execCommand(`tar -xzvf ${dest}/sqlite.tar.gz -C ${dest}`);
+	await utils.execCommand(`tar -xzvf ${dest}/amalgamation.tar.gz -C ${dest}`);
+}
+
+async function main() {
+	const rootDir = utils.rootDir();
+	const dest = `${rootDir}/packages/app-mobile/lib/sql-extensions`;
+
+	try {
+		await fs.ensureDir(dest);
+
+		if (utils.isLinux()) {
+			try {
+				await fs.promises.access(`${dest}/spellfix.so`);
+			} catch (e) {
+				await getSourceCode(dest);
+				await utils.execCommand(`gcc -shared -fPIC -Wall -I${dest}/sqlite-autoconf-3330000/ ${dest}/sqlite/ext/misc/spellfix.c -o ${dest}/spellfix.so`);
+			}
+		}
+
+		if (utils.isMac()) {
+			try {
+				await fs.promises.access(`${dest}/spellfix.dylib`);
+			} catch (e) {
+				await getSourceCode(dest);
+				await utils.execCommand(`gcc -shared -fPIC -Wall -I${dest}/sqlite-autoconf-3330000/ -dynamiclib ${dest}/sqlite/ext/misc/spellfix.c -o ${dest}/spellfix.dylib`);
+			}
+		}
+
+		// if (utils.isWindows()) {
+		// 	try {
+		// 		await fs.promises.access(`${dest}/spellfix.dll`);
+		// 	} catch (e) {
+		// 		await getSourceCode(dest);
+		// 		await utils.execCommand(`cl /I ${dest}/sqlite-autoconf-3330000 ${dest}/sqlite/ext/misc/spellfix.c -link -dll -out:spellfix.dll `)
+		// 	}
+		// }
+	} catch (e) {
+		console.warn(e);
+	}
+
+	await fs.remove(`${dest}/sqlite.tar.gz`);
+	await fs.remove(`${dest}/amalgamation.tar.gz`);
+	await fs.remove(`${dest}/sqlite`);
+	await fs.remove(`${dest}/sqlite-autoconf-3330000`);
+}
+
+module.exports = main;

packages/tools/gulp/tasks/copyLib.js

@@ -0,0 +1,17 @@
+const utils = require('../utils');
+
+const rootDir = utils.rootDir();
+
+module.exports = {
+	src: `${rootDir}/packages/app-mobile/lib/**/*`,
+	fn: async function() {
+		const copyOptions = {
+			excluded: [
+				`${rootDir}/packages/renderer/node_modules`,
+			],
+		};
+
+		await utils.copyDir(`${rootDir}/packages/app-mobile/lib`, `${rootDir}/packages/app-cli/build/lib`, copyOptions);
+		await utils.copyDir(`${rootDir}/packages/app-mobile/lib`, `${rootDir}/packages/app-desktop/lib`, copyOptions);
+	},
+};

packages/tools/gulp/tasks/deleteBuildDirs.js

@@ -0,0 +1,11 @@
+const utils = require('../utils');
+const rootDir = utils.rootDir();
+const fs = require('fs-extra');
+
+module.exports = {
+	src: '',
+	fn: async function() {
+		await fs.remove(`${rootDir}/packages/app-cli/tests-build`);
+		await fs.remove(`${rootDir}/packages/app-cli/build`);
+	},
+};

packages/tools/gulp/tasks/tsc.js

@@ -0,0 +1,20 @@
+const execa = require('execa');
+
+const rootDir = `${__dirname}/../../../`;
+process.chdir(rootDir);
+
+module.exports = {
+	src: [
+		'packages/app-mobile/**/*.tsx',
+		'packages/app-mobile/**/*.ts',
+		'packages/app-desktop/**/*.tsx',
+		'packages/app-desktop/**/*.ts',
+		'packages/app-cli/**/*.tsx',
+		'packages/app-cli/**/*.ts',
+	],
+	fn: async function() {
+		const promise = execa('node', ['node_modules/typescript/bin/tsc', '--project', 'tsconfig.json'], { cwd: rootDir });
+		promise.stdout.pipe(process.stdout);
+		return promise;
+	},
+};

packages/tools/gulp/tasks/updateIgnoredTypeScriptBuild.js

@@ -0,0 +1,64 @@
+const utils = require('../utils');
+const rootDir = utils.rootDir();
+
+module.exports = {
+	src: '',
+	fn: async function() {
+		// Note that patterns must be `folder/**` in order to make it skip the
+		// directory. If it's `folder/**/*` it will filter the results but still
+		// scan the directory.
+		//
+		// https://github.com/isaacs/node-glob/issues/371
+
+		const tsFiles = utils.globSync('{**/*.ts,**/*.tsx}', {
+			cwd: rootDir,
+			ignore: [
+				'**/.git/**',
+				'**/node_modules/**',
+				'Assets/**',
+				'docs/**',
+				'packages/app-cli/build/**',
+				'packages/app-cli/tests-build/**',
+				'packages/app-cli/tests/html_to_md/**',
+				'packages/app-cli/tests/md_to_html/**',
+				'packages/app-cli/tests/support/plugins/**',
+				'packages/app-cli/tests/support/syncTargetSnapshots/**',
+				'packages/app-cli/tests/sync/**',
+				'packages/app-cli/tests/test data/**',
+				'packages/app-cli/tests/tmp/**',
+				'packages/app-clipper/popup/build/**',
+				'packages/app-desktop/dist/**',
+				'packages/app-desktop/dist/**',
+				'packages/app-mobile/android/**',
+				'packages/app-mobile/ios/**',
+				'packages/default-plugins/plugin-sources/**',
+				'packages/generator-joplin/generators/app/templates/script/publish/**',
+				'packages/fork-sax/**',
+				'packages/lib/plugin_types/**',
+				'packages/server/**',
+				'packages/utils/**',
+				'packages/transcribe/**',
+			],
+		}).filter(f => !f.endsWith('.d.ts'));
+
+		// Use this to check what files are being scanned.
+		// Also change glob call to `glob.sync('**/*',`
+		//
+		// const fs = require('fs-extra');
+		// fs.writeFileSync('/Users/laurent/listfile.txt', JSON.stringify(tsFiles, null, '\t'));
+
+		const ignoredJsFiles = tsFiles.map(f => {
+			const s = f.split('.');
+			s.pop();
+			return `${s.join('.')}.js`;
+		});
+
+		const regex = /(# AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD)[\s\S]*(# AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD)/;
+		const replacement = `$1\n${ignoredJsFiles.join('\n')}\n$2`;
+
+		await Promise.all([
+			utils.replaceFileText(`${rootDir}/.gitignore`, regex, replacement),
+			utils.replaceFileText(`${rootDir}/.ignore.eslint`, regex, replacement),
+		]);
+	},
+};

packages/tools/gulp/tasks/updateIgnoredTypeScriptBuildRun.js

@@ -0,0 +1,12 @@
+// Allow running that task "updateIgnoredTypeScriptBuild" without gulp
+
+const task = require('./updateIgnoredTypeScriptBuild.js');
+
+const main = async () => {
+	await task.fn();
+};
+
+main().catch((error) => {
+	console.error(error);
+	process.exit(1);
+});

packages/tools/gulp/utils.js

@@ -0,0 +1,228 @@
+const fs = require('fs-extra');
+const execa = require('execa');
+const glob = require('glob');
+
+const utils = {};
+
+utils.isLinux = () => {
+	return process && process.platform === 'linux';
+};
+
+utils.isWindows = () => {
+	return process && process.platform === 'win32';
+};
+
+utils.isMac = () => {
+	return process && process.platform === 'darwin';
+};
+
+utils.execCommandVerbose = function(commandName, args = []) {
+	console.info(`> ${commandName}`, args && args.length ? args : '');
+	const promise = execa(commandName, args);
+	promise.stdout.pipe(process.stdout);
+	promise.stderr.pipe(process.stderr);
+	return promise;
+};
+
+utils.execCommand = function(command) {
+	const exec = require('child_process').exec;
+
+	return new Promise((resolve, reject) => {
+		exec(command, { maxBuffer: 1024 * 1024 }, (error, stdout) => {
+			if (error) {
+
+				// Special case for robocopy, which will return non-zero error codes
+				// when sucessful. Doc is very imprecise but <= 7 seems more or less
+				// fine and >= 8 seems more errorish. https://ss64.com/nt/robocopy-exit.html
+				if (command.indexOf('robocopy') === 0 && error.code <= 7) {
+					resolve(stdout.trim());
+					return;
+				}
+
+				if (error.signal === 'SIGTERM') {
+					resolve('Process was killed');
+				} else {
+					const newError = new Error(`Code: ${error.code}: ${error.message}: ${stdout.trim()}`);
+					reject(newError);
+				}
+			} else {
+				resolve(stdout.trim());
+			}
+		});
+	});
+};
+
+utils.dirname = function(path) {
+	if (!path) throw new Error('Path is empty');
+	const s = path.split(/\/|\\/);
+	s.pop();
+	return s.join('/');
+};
+
+utils.basename = function(path) {
+	if (!path) throw new Error('Path is empty');
+	const s = path.split(/\/|\\/);
+	return s[s.length - 1];
+};
+
+utils.filename = function(path, includeDir = false) {
+	if (!path) throw new Error('Path is empty');
+	let output = includeDir ? path : utils.basename(path);
+	if (output.indexOf('.') < 0) return output;
+
+	output = output.split('.');
+	output.pop();
+	return output.join('.');
+};
+
+utils.toSystemSlashes = function(path) {
+	if (utils.isWindows()) return path.replace(/\//g, '\\');
+	return path.replace(/\\/g, '/');
+};
+
+utils.fileExtension = function(path) {
+	if (!path) throw new Error('Path is empty');
+
+	const output = path.split('.');
+	if (output.length <= 1) return '';
+	return output[output.length - 1];
+};
+
+utils.replaceFileText = async function(filePath, regex, toInsert) {
+	const content = await fs.readFile(filePath, 'utf8');
+	const newContent = content.replace(regex, toInsert);
+	if (newContent === content) return Promise.resolve();
+	await fs.writeFile(filePath, newContent);
+};
+
+utils.copyDir = async function(src, dest, options) {
+	options = { excluded: [],
+		delete: true, ...options };
+
+	src = utils.toSystemSlashes(src);
+	dest = utils.toSystemSlashes(dest);
+
+	await utils.mkdir(dest);
+
+	if (utils.isWindows()) {
+		let cmd = ['robocopy'];
+		cmd.push(`"${src}"`);
+		cmd.push(`"${dest}"`);
+		cmd.push('/e');
+		cmd.push('/r:0');
+		if (options.delete) cmd.push('/purge');
+
+		if (options.excluded.length) {
+			cmd.push('/xd');
+			cmd = cmd.concat(options.excluded.map(p => `"${utils.toSystemSlashes(p)}"`).join(' '));
+		}
+
+		await utils.execCommand(cmd.join(' '));
+	} else {
+		let excludedFlag = '';
+		if (options.excluded.length) {
+			excludedFlag = options.excluded.map(f => {
+				return `--exclude "${f}"`;
+			}).join(' ');
+		}
+
+		let deleteFlag = '';
+		if (options.delete) deleteFlag = '--delete';
+
+		await utils.execCommand(`rsync -a ${deleteFlag} ${excludedFlag} "${src}/" "${dest}/"`);
+	}
+};
+
+// Occasionally, fs.mkdirp throws a "EEXIST" error if the directory already
+// exists, while it should actually ignore the error. So we have this wrapper
+// that actually handle the error. It means in general this method should be
+// preferred to avoid random failures on CI or when building the app.
+//
+// https://github.com/laurent22/joplin/issues/6935#issuecomment-1274404470
+utils.mkdir = async function(dir) {
+	if (utils.isWindows()) {
+		return utils.execCommand(`if not exist "${utils.toSystemSlashes(dir)}" mkdir "${utils.toSystemSlashes(dir)}"`);
+	} else {
+		try {
+			// Can't return right away, or the exception won't be caught
+			const result = await fs.mkdirp(dir);
+			return result;
+		} catch (error) {
+			// Shouldn't happen but sometimes does. So we ignore the error in
+			// this case.
+			if (error.code === 'EEXIST') return;
+			throw error;
+		}
+	}
+};
+
+utils.mkdirp = async function(dir) {
+	return utils.mkdir(dir);
+};
+
+utils.copyFile = async function(src, dest) {
+	await fs.copy(src, dest);
+};
+
+utils.rootDir = function() {
+	return utils.dirname(utils.dirname(utils.dirname(__dirname)));
+};
+
+utils.registerGulpTasks = function(gulp, tasks) {
+	for (const taskName in tasks) {
+		gulp.task(taskName, tasks[taskName].fn);
+	}
+};
+
+utils.setPackagePrivateField = async function(filePath, value) {
+	const text = await fs.readFile(filePath, 'utf8');
+	const obj = JSON.parse(text);
+	if (!value) {
+		delete obj.private;
+	} else {
+		obj.private = true;
+	}
+	await fs.writeFile(filePath, JSON.stringify(obj, null, 2), 'utf8');
+};
+
+utils.insertContentIntoFile = async (filePath, marker, contentToInsert, createIfNotExist = false) => {
+	const fs = require('fs-extra');
+	const fileExists = await fs.pathExists(filePath);
+
+	if (!fileExists) {
+		if (!createIfNotExist) throw new Error(`File not found: ${filePath}`);
+		await fs.writeFile(filePath, `${marker}\n${contentToInsert}\n${marker}`);
+	} else {
+		let content = await fs.readFile(filePath, 'utf-8');
+		// [^]* matches any character including new lines
+		const regex = new RegExp(`${marker}[^]*?${marker}`);
+		content = content.replace(regex, `${marker}\n${contentToInsert}\n${marker}`);
+		await fs.writeFile(filePath, content);
+	}
+};
+
+utils.getFilename = (path) => {
+	const lastPart = path.split('/').pop();
+	if (lastPart.indexOf('.') < 0) return lastPart;
+
+	const splitted = lastPart.split('.');
+	splitted.pop();
+	return splitted.join('.');
+};
+
+utils.msleep = (ms) => {
+	return new Promise((resolve) => {
+		setTimeout(() => {
+			resolve();
+		}, ms);
+	});
+};
+
+utils.globSync = (pattern, options = {}) => {
+	let output = glob.sync(pattern, options);
+	output = output.map(f => f.replace(/\\/g, '/'));
+	output.sort();
+	return output;
+};
+
+module.exports = utils;