feat: Initial clean release of CHM

packages/tools/website/processDocs.ts

@@ -0,0 +1,531 @@
+/* eslint-disable import/prefer-default-export */
+
+import { execCommand, getRootDir } from '@joplin/utils';
+import { readFile, readdir, stat, writeFile } from 'fs/promises';
+import MarkdownIt from 'markdown-it';
+import { htmlentities, isSelfClosingTag } from '@joplin/utils/html';
+import { compileWithFrontMatter, stripOffFrontMatter } from './utils/frontMatter';
+import StateCore = require('markdown-it/lib/rules_core/state_core');
+import Token = require('markdown-it/lib/token');
+import { copy, mkdirp, remove, pathExists } from 'fs-extra';
+import { basename, dirname } from 'path';
+import markdownUtils, { MarkdownTable } from '@joplin/lib/markdownUtils';
+import { readmeFileTitle } from './utils/parser';
+import { chdir } from 'process';
+import yargs = require('yargs');
+import { extractOpenGraphTags } from './utils/openGraph';
+
+import md5File = require('md5-file');
+import * as htmlparser2 from '@joplin/fork-htmlparser2';
+import styleToJs = require('style-to-js');
+import * as crypto from 'crypto';
+
+interface Config {
+	baseUrl: string;
+	docusaurusBuildEnabled: boolean;
+	docusaurusBuildCommand: string;
+}
+
+interface List {
+	type: 'bullet' | 'number';
+}
+
+interface Context {
+	currentTable?: MarkdownTable;
+	currentTableContent?: string[];
+	currentTableCellIndex?: number;
+	inHeader?: boolean;
+	listStack?: List[];
+	listStarting?: boolean;
+	currentLinkAttrs?: [string, string][] | null;
+	inFence?: boolean;
+	fenceStarting?: string;
+	processedFiles?: string[];
+	isNews?: boolean;
+	donateLinks?: string;
+	inQuote?: boolean;
+}
+
+const configs: Record<string, Config> = {
+	dev: {
+		baseUrl: 'http://localhost:8077',
+		docusaurusBuildCommand: 'buildDev',
+		docusaurusBuildEnabled: true,
+	},
+	prod: {
+		baseUrl: 'https://joplinapp.org',
+		docusaurusBuildCommand: '_build',
+		docusaurusBuildEnabled: true,
+	},
+};
+
+const md5 = (s: string) => {
+	return crypto.createHash('md5').update(s).digest('hex');
+};
+
+const parseHtml = (html: string) => {
+	const output: string[] = [];
+
+	interface Tag {
+		name: string;
+	}
+
+	const tagStack: Tag[] = [];
+
+	const currentTag = () => {
+		if (!tagStack.length) return { name: '', processed: false };
+		return tagStack[tagStack.length - 1];
+	};
+
+	const parser = new htmlparser2.Parser({
+
+		onopentag: (name: string, attrs: Record<string, string>) => {
+			tagStack.push({ name });
+
+			const closingSign = isSelfClosingTag(name) ? '/>' : '>';
+
+			const attrHtml: string[] = [];
+
+			for (const n in attrs) {
+				if (!attrs.hasOwnProperty(n)) continue;
+				let escapedValue = `"${htmlentities(attrs[n])}"`;
+				if (n === 'style') {
+					escapedValue = `{${JSON.stringify(styleToJs(attrs.style))}}`;
+				}
+				attrHtml.push(`${n}=${escapedValue}`);
+			}
+
+			const closingSpace = isSelfClosingTag(name) || !!attrHtml.length ? ' ' : '';
+
+			output.push(`<${name}${attrHtml.length ? ` ${attrHtml.join(' ')}` : ''}${closingSpace}${closingSign}`);
+		},
+
+		ontext: (decodedText: string) => {
+			output.push(htmlentities(decodedText));
+		},
+
+		onclosetag: (name: string) => {
+			const current = currentTag();
+
+			if (current.name === name) tagStack.pop();
+
+			if (isSelfClosingTag(name)) return;
+			output.push(`</${name}>`);
+		},
+
+	}, { decodeEntities: true });
+
+	parser.write(html);
+	parser.end();
+
+	return output.join('');
+};
+
+const escapeForMdx = (s: string): string => {
+	return s
+		.replace(/</g, '&lt;')
+		.replace(/>/g, '&gt;')
+		.replace(/`/g, '\\`');
+};
+
+const paragraphBreak = '///PARAGRAPH_BREAK///';
+const blockQuoteStart = '///BLOCK_QUOTE_START///';
+const blockQuoteEnd = '///BLOCK_QUOTE_END///';
+
+const processToken = (token: Token, output: string[], context: Context): void => {
+	if (!context.listStack) context.listStack = [];
+
+	let contentProcessed = false;
+	const type = token.type as string;
+
+	const currentList = context.listStack.length ? context.listStack[context.listStack.length - 1] : null;
+
+	const doEscapeForMdx = (text: string) => {
+		if (context.inFence) return text;
+		return escapeForMdx(text);
+	};
+
+	const content: string[] = [];
+
+	if (type === 'heading_open') {
+		content.push(paragraphBreak);
+		content.push(`${token.markup} `);
+	} else if (type === 'heading_close') {
+		content.push(paragraphBreak);
+	} else if (type === 'paragraph_open' || type === 'paragraph_close') {
+		if (!currentList) {
+			content.push(paragraphBreak);
+		}
+	} else if (type === 'fence') {
+		context.inFence = true;
+		context.fenceStarting = token.markup;
+		content.push(`${token.markup}${token.info || ''}\n`);
+	} else if (type === 'html_block') {
+		contentProcessed = true;
+		content.push(parseHtml(token.content.trim()));
+	} else if (type === 'html_inline') {
+		contentProcessed = true;
+		content.push(parseHtml(token.content.trim()));
+	} else if (type === 'code_inline') {
+		contentProcessed = true;
+		content.push(`\`${token.content}\``);
+	} else if (type === 'code_block') {
+		contentProcessed = true;
+		content.push(`\`\`\`\n${token.content}\`\`\``);
+		content.push(paragraphBreak);
+	} else if (type === 'strong_open' || type === 'strong_close') {
+		content.push('**');
+	} else if (type === 'em_open' || type === 'em_close') {
+		content.push('*');
+	} else if (type === 'blockquote_open' || type === 'blockquote_close') {
+		content.push(type === 'blockquote_open' ? blockQuoteStart : blockQuoteEnd);
+	} else if (type === 'table_open') {
+		context.currentTable = {
+			header: [],
+			rows: [],
+		};
+		context.currentTableContent = [];
+	} else if (type === 'th_open') {
+		context.currentTable.header.push({
+			label: '',
+			name: `${context.currentTable.header.length}`,
+			disableHtmlEscape: true,
+		});
+	} else if (type === 'thead_open') {
+		context.inHeader = true;
+	} else if (type === 'thead_close') {
+		context.inHeader = false;
+	} else if (type === 'th_close') {
+		context.currentTable.header[context.currentTable.header.length - 1].label = context.currentTableContent.join('');
+		context.currentTableContent = [];
+	} else if (type === 'tr_open') {
+		if (!context.inHeader) {
+			context.currentTable.rows.push({});
+			context.currentTableCellIndex = 0;
+		}
+	} else if (type === 'td_open') {
+		const row = context.currentTable.rows[context.currentTable.rows.length - 1];
+		row[`${context.currentTableCellIndex}`] = '';
+	} else if (type === 'td_close') {
+		const row = context.currentTable.rows[context.currentTable.rows.length - 1];
+		row[`${context.currentTableCellIndex}`] = context.currentTableContent.join('');
+		context.currentTableCellIndex++;
+		context.currentTableContent = [];
+	} else if (type === 'table_close') {
+		const tableMd = markdownUtils.createMarkdownTable(context.currentTable.header, context.currentTable.rows);
+		content.push(paragraphBreak);
+		content.push(tableMd);
+		content.push(paragraphBreak);
+		context.currentTable = null;
+	} else if (type === 'bullet_list_open') {
+		context.listStarting = !context.listStack.length;
+		context.listStack.push({ type: 'bullet' });
+	} else if (type === 'ordered_list_open') {
+		context.listStarting = !context.listStack.length;
+		context.listStack.push({ type: 'number' });
+	} else if (type === 'bullet_list_close' || type === 'ordered_list_close') {
+		context.listStack.pop();
+		if (!context.listStack.length) content.push(paragraphBreak);
+	} else if (type === 'list_item_open') {
+		if (!context.listStarting) content.push('\n');
+		context.listStarting = false;
+		const indent = '\t'.repeat((context.listStack.length - 1));
+		if (currentList.type === 'bullet') content.push(`${indent}- `);
+		if (currentList.type === 'number') content.push(`${indent + token.info}. `);
+	} else if (type === 'image') {
+		(context.currentTable ? context.currentTableContent : content).push('![');
+	} else if (type === 'link_open') {
+		content.push('[');
+		context.currentLinkAttrs = token.attrs;
+	} else if (type === 'link_close') {
+		const href = context.currentLinkAttrs.find((a: string[]) => a[0] === 'href')[1];
+		content.push(`](${escapeForMdx(href)})`);
+		context.currentLinkAttrs = null;
+	}
+
+	if (token.children) {
+		for (const child of token.children) {
+			processToken(child, content, context);
+		}
+	} else if (token.content && !contentProcessed) {
+		content.push(doEscapeForMdx(token.content));
+	}
+
+	if (type === 'fence') {
+		content.push(context.fenceStarting);
+		content.push(paragraphBreak);
+		context.inFence = false;
+	}
+
+	if (type === 'image') {
+		let src: string = token.attrs.find((a: string[]) => a[0] === 'src')[1];
+		if (!src.startsWith('http') && !src.startsWith('/')) src = `/${src}`;
+		content.push(`](${doEscapeForMdx(src)})`);
+	}
+
+	for (const c of content) {
+		if (context.currentTable) {
+			context.currentTableContent.push(c);
+		} else {
+			output.push(c);
+		}
+	}
+};
+
+const resolveParagraphBreaks = (output: string[]): string[] => {
+	if (!output.length) return output;
+
+	while (output.length && output[0] === paragraphBreak) {
+		output.splice(0, 1);
+	}
+
+	while (output.length && output[output.length - 1] === paragraphBreak) {
+		output.pop();
+	}
+
+	let previous = '';
+	const newOutput: string[] = [];
+	for (const s of output) {
+		if (s === paragraphBreak && previous === paragraphBreak) continue;
+		newOutput.push(s === paragraphBreak ? '\n\n' : s);
+		previous = s;
+	}
+
+	return newOutput;
+};
+
+const resolveBlockQuotes = (output: string): string => {
+	if (!output) return output;
+
+	const lines = output.split('\n');
+	let inQuotes = false;
+	const quotedLines: string[] = [];
+	const newOutput: string[] = [];
+	for (const line of lines) {
+		if (line === blockQuoteStart) {
+			inQuotes = true;
+		} else if (line === blockQuoteEnd) {
+			inQuotes = false;
+			const s = quotedLines.join('\n').trim().split('\n').map(l => `> ${l}`).join('\n');
+			newOutput.push(s);
+		} else if (inQuotes) {
+			quotedLines.push(line);
+		} else {
+			newOutput.push(line);
+		}
+	}
+
+	return newOutput.join('\n');
+};
+
+export const processUrls = (md: string) => {
+	md = md
+		.replace(/https:\/\/github.com\/laurent22\/joplin\/blob\/dev\/readme\/(.*?)\/index\.md/g, '/help/$1')
+		.replace(/https:\/\/github.com\/laurent22\/joplin\/blob\/dev\/readme\/(.*?)\.md/g, '/help/$1');
+
+	return md;
+};
+
+export const processMarkdownDoc = (sourceContent: string, context: Context): string => {
+	const markdownIt = new MarkdownIt({
+		breaks: true,
+		linkify: true,
+		html: true,
+	});
+
+	let items: string[] = [];
+
+	markdownIt.core.ruler.push('converter', (state: StateCore) => {
+		const tokens = state.tokens;
+		// console.info(JSON.stringify(tokens, null, '\t'));
+		for (let i = 0; i < tokens.length; i++) {
+			const token = tokens[i];
+			processToken(token, items, context);
+		}
+	});
+
+	const mdAndFrontMatter = stripOffFrontMatter(sourceContent);
+
+	markdownIt.render(mdAndFrontMatter.doc);
+
+	items = resolveParagraphBreaks(items);
+
+	const output = resolveBlockQuotes(processUrls(items.join('').trim()));
+
+	return compileWithFrontMatter({
+		...mdAndFrontMatter,
+		doc: output,
+	});
+};
+
+const processMarkdownFile = async (sourcePath: string, destPath: string, context: Context) => {
+	context.processedFiles.push(destPath);
+
+	const sourceContent = await readFile(sourcePath, 'utf-8');
+	const destContent = processMarkdownDoc(sourceContent, context);
+	await mkdirp(dirname(destPath));
+
+	const title = await readmeFileTitle(sourcePath);
+
+	const mdAndFrontMatter = stripOffFrontMatter(destContent);
+	mdAndFrontMatter.header.sidebar_label = title;
+
+	if (context.isNews) {
+		const dateString = basename(sourcePath).split('-')[0];
+		const formatted = `${dateString.substring(0, 4)}-${dateString.substring(4, 6)}-${dateString.substring(6, 8)}`;
+		mdAndFrontMatter.header.date = formatted;
+	}
+
+	const og = await extractOpenGraphTags(sourcePath);
+	mdAndFrontMatter.header.title = og.title;
+	mdAndFrontMatter.header.description = og.description;
+	if (og.image) mdAndFrontMatter.header.image = og.image;
+
+	mdAndFrontMatter.doc = mdAndFrontMatter.doc.replace(/^# (.*)\n/, `# $1\n\n${context.donateLinks}\n\n`);
+
+	if (mdAndFrontMatter.header.forum_url) {
+		mdAndFrontMatter.doc += `\n\n* * *\n\n[<icon icon="fa-brands fa-discourse" size="lg" /> Discuss on the forum](${mdAndFrontMatter.header.forum_url})`;
+	}
+
+	const fullContent = compileWithFrontMatter(mdAndFrontMatter);
+
+	if (await pathExists(destPath)) {
+		const existingMd5 = await md5File(destPath);
+		const newMd5 = md5(fullContent);
+		if (existingMd5 === newMd5) return;
+	}
+
+	await writeFile(destPath, fullContent, 'utf-8');
+};
+
+const processDocFiles = async (sourceDir: string, destDir: string, excluded: string[], context: Context) => {
+	if (!context.processedFiles) context.processedFiles = [];
+
+	await mkdirp(destDir);
+
+	const files = await readdir(sourceDir);
+	for (const file of files) {
+		const fullPath = `${sourceDir}/${file}`;
+		if (excluded.includes(fullPath)) continue;
+
+		const info = await stat(fullPath);
+		const destPath = `${destDir}/${file}`;
+
+		if (info.isDirectory()) {
+			await processDocFiles(fullPath, destPath, excluded, context);
+		} else {
+			if (file.endsWith('.md')) {
+				console.info(`Process: ${fullPath}`);
+				await processMarkdownFile(fullPath, destPath, context);
+			} else if (file === '_category_.yml') {
+				context.processedFiles.push(destPath);
+				await copy(fullPath, destPath);
+			}
+		}
+	}
+};
+
+export const deleteUnprocessedFiles = async (dirPath: string, processedFiles: string[]) => {
+	const files = await readdir(dirPath);
+	for (const file of files) {
+		const fullPath = `${dirPath}/${file}`;
+		const info = await stat(fullPath);
+
+		if (info.isDirectory()) {
+			await deleteUnprocessedFiles(fullPath, processedFiles);
+		} else {
+			if (!processedFiles.includes(fullPath)) {
+				console.info(`Delete: ${fullPath}`);
+				await remove(fullPath);
+			}
+		}
+	}
+};
+
+const copyFile = async (sourceFile: string, destFile: string) => {
+	console.info(`Copy: ${destFile}`);
+	await copy(sourceFile, destFile);
+};
+
+const getDonateLinks = () => {
+	return `<div className="donate-links">
+
+[![Donate using PayPal](https://raw.githubusercontent.com/laurent22/joplin/dev/Assets/WebsiteAssets/images/badges/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?hosted_button_id=WQCERTSSLCC7U) [![Sponsor on GitHub](https://raw.githubusercontent.com/laurent22/joplin/dev/Assets/WebsiteAssets/images/badges/GitHub-Badge.svg)](https://github.com/sponsors/laurent22/) [![Become a patron](https://raw.githubusercontent.com/laurent22/joplin/dev/Assets/WebsiteAssets/images/badges/Patreon-Badge.svg)](https://www.patreon.com/joplin) [![Donate using IBAN](https://raw.githubusercontent.com/laurent22/joplin/dev/Assets/WebsiteAssets/images/badges/Donate-IBAN.svg)](https://joplinapp.org/donate/#donations)
+
+</div>`;
+};
+
+export const buildDocusaurus = async (docBuilderDir: string, buildCommand: string, baseUrl: string) => {
+	chdir(docBuilderDir);
+	await execCommand(`yarn ${buildCommand}`, {
+		env: {
+			WEBSITE_BASE_URL: baseUrl,
+		},
+	});
+};
+
+async function main() {
+	const argv = await yargs.argv;
+	const env = argv.env as string;
+
+	if (!env) throw new Error('Env must be specified: either "dev" or "prod"');
+
+	const config = { ...configs[env] };
+
+	if ('docuBuild' in argv && !argv.docuBuild) config.docusaurusBuildEnabled = false;
+
+	const rootDir = await getRootDir();
+	const docBuilderDir = `${rootDir}/packages/doc-builder`;
+	const destHelpDir = `${docBuilderDir}/help`;
+	const newsDestDir = `${docBuilderDir}/news`;
+	const readmeDir = `${rootDir}/readme`;
+
+	const donateLinks = getDonateLinks();
+
+	const mainContext: Context = { donateLinks };
+
+	await processDocFiles(readmeDir, destHelpDir, [
+		`${readmeDir}/_i18n`,
+		`${readmeDir}/i18n`,
+		`${readmeDir}/cla.md`,
+		`${readmeDir}/download.md`,
+		`${readmeDir}/faq_joplin_cloud.md`,
+		`${readmeDir}/privacy.md`,
+		`${readmeDir}/donate.md`,
+		`${readmeDir}/connection_check.md`,
+		`${readmeDir}/licenses.md`,
+		`${readmeDir}/welcome`,
+		`${readmeDir}/news`,
+	], mainContext);
+
+	await deleteUnprocessedFiles(destHelpDir, mainContext.processedFiles);
+
+	const newsContext: Context = { isNews: true, donateLinks };
+	await processDocFiles(`${readmeDir}/news`, newsDestDir, [], newsContext);
+	await deleteUnprocessedFiles(newsDestDir, newsContext.processedFiles);
+
+	if (await pathExists(`${readmeDir}/i18n`)) {
+		const localeContext: Context = { donateLinks };
+		await processDocFiles(`${readmeDir}/i18n`, `${docBuilderDir}/i18n`, [], localeContext);
+		await deleteUnprocessedFiles(`${docBuilderDir}/i18n`, localeContext.processedFiles);
+	} else {
+		console.info('i18n folder is missing - skipping it');
+	}
+
+	await copyFile(`${rootDir}/Assets/WebsiteAssets/images`, `${docBuilderDir}/static/images`);
+
+	if (config.docusaurusBuildEnabled) {
+		await buildDocusaurus(docBuilderDir, config.docusaurusBuildCommand, config.baseUrl);
+	} else {
+		console.info('Skipping Docusaurus build...');
+	}
+}
+
+if (require.main === module) {
+	// eslint-disable-next-line promise/prefer-await-to-then
+	main().catch((error) => {
+		console.error('Fatal error');
+		console.error(error);
+		process.exit(1);
+	});
+}