feat: Initial clean release of CHM

packages/lib/services/interop/InteropService.ts

@@ -0,0 +1,565 @@
+import { ModuleType, FileSystemItem, ImportModuleOutputFormat, ImportOptions, ExportOptions, ImportExportResult, ExportProgressState, ExportModuleOutputFormat } from './types';
+import shim from '../../shim';
+import { _, _n } from '../../locale';
+import BaseItem from '../../models/BaseItem';
+import BaseModel, { ModelType } from '../../BaseModel';
+import Resource from '../../models/Resource';
+import Folder from '../../models/Folder';
+import NoteTag from '../../models/NoteTag';
+import Note from '../../models/Note';
+import * as ArrayUtils from '../../ArrayUtils';
+import isNoteLockEnabled from '../noteLock/isNoteLockEnabled';
+import NoteLockNote, { NoteLockNoteEntity } from '../noteLock/NoteLockNote';
+import { DecryptedNoteLockKey } from '../noteLock/NoteLockKey';
+import NoteLockSession from '../noteLock/NoteLockSession';
+import { NoteEntity } from '../database/types';
+import InteropService_Importer_Jex from './InteropService_Importer_Jex';
+import InteropService_Importer_Md from './InteropService_Importer_Md';
+import InteropService_Importer_Md_frontmatter from './InteropService_Importer_Md_frontmatter';
+import InteropService_Importer_Obsidian from './InteropService_Importer_Obsidian';
+import InteropService_Importer_Raw from './InteropService_Importer_Raw';
+import InteropService_Exporter_Jex from './InteropService_Exporter_Jex';
+import InteropService_Exporter_Raw from './InteropService_Exporter_Raw';
+import InteropService_Exporter_Md from './InteropService_Exporter_Md';
+import InteropService_Exporter_Md_frontmatter from './InteropService_Exporter_Md_frontmatter';
+import InteropService_Importer_Base from './InteropService_Importer_Base';
+import InteropService_Exporter_Base from './InteropService_Exporter_Base';
+import Module, { makeExportModule, makeImportModule } from './Module';
+import InteropService_Exporter_Html from './InteropService_Exporter_Html';
+import InteropService_Importer_EnexToHtml from './InteropService_Importer_EnexToHtml';
+import InteropService_Importer_EnexToMd from './InteropService_Importer_EnexToMd';
+import InteropService_Importer_OneNote from './InteropService_Importer_OneNote';
+const { sprintf } = require('sprintf-js');
+import { fileExtension } from '../../path-utils';
+import { EventEmitter } from 'events';
+
+export default class InteropService {
+
+	private defaultModules_: Module[];
+	private userModules_: Module[] = [];
+	private eventEmitter_: EventEmitter = null;
+	private static instance_: InteropService;
+	private domParser_: DOMParser;
+	private xmlSerializer_: XMLSerializer;
+
+	public static instance(): InteropService {
+		if (!this.instance_) this.instance_ = new InteropService();
+		return this.instance_;
+	}
+
+	public constructor() {
+		this.eventEmitter_ = new EventEmitter();
+	}
+
+	public on(eventName: string, callback: (...args: unknown[])=> void) {
+		return this.eventEmitter_.on(eventName, callback);
+	}
+
+	public off(eventName: string, callback: (...args: unknown[])=> void) {
+		return this.eventEmitter_.removeListener(eventName, callback);
+	}
+
+	public modules() {
+		if (!this.defaultModules_) {
+			const importModules = [
+				makeImportModule({
+					format: 'jex',
+					fileExtensions: ['jex'],
+					sources: [FileSystemItem.File],
+					description: _('Joplin Export File'),
+				}, () => new InteropService_Importer_Jex()),
+
+				makeImportModule({
+					format: 'raw',
+					sources: [FileSystemItem.Directory],
+					description: _('Joplin Export Directory'),
+					separatorAfter: true,
+				}, () => new InteropService_Importer_Raw()),
+
+				makeImportModule({
+					format: 'enex',
+					fileExtensions: ['enex'],
+					sources: [FileSystemItem.File],
+					description: _('Evernote Export File (as HTML)'),
+					supportsMobile: false,
+					outputFormat: ImportModuleOutputFormat.Html,
+				}, () => new InteropService_Importer_EnexToHtml()),
+
+				makeImportModule({
+					format: 'enex',
+					fileExtensions: ['enex'],
+					sources: [FileSystemItem.File],
+					description: _('Evernote Export File (as Markdown)'),
+					supportsMobile: false,
+					isDefault: true,
+				}, () => new InteropService_Importer_EnexToMd()),
+
+				makeImportModule({
+					format: 'enex',
+					fileExtensions: ['enex'],
+					sources: [FileSystemItem.Directory],
+					description: _('Evernote Export Files (Directory, as HTML)'),
+					supportsMobile: false,
+					outputFormat: ImportModuleOutputFormat.Html,
+				}, () => new InteropService_Importer_EnexToHtml()),
+
+				makeImportModule({
+					format: 'enex',
+					fileExtensions: ['enex'],
+					sources: [FileSystemItem.Directory],
+					description: _('Evernote Export Files (Directory, as Markdown)'),
+					supportsMobile: false,
+				}, () => new InteropService_Importer_EnexToMd()),
+
+				makeImportModule({
+					format: 'html',
+					fileExtensions: ['html'],
+					sources: [FileSystemItem.File, FileSystemItem.Directory],
+					isNoteArchive: false, // Tells whether the file can contain multiple notes (eg. Enex or Jex format)
+					description: _('HTML document'),
+				}, () => new InteropService_Importer_Md()),
+
+				makeImportModule({
+					format: 'md',
+					fileExtensions: ['md', 'markdown', 'txt', 'html'],
+					sources: [FileSystemItem.File, FileSystemItem.Directory],
+					isNoteArchive: false, // Tells whether the file can contain multiple notes (eg. Enex or Jex format)
+					description: _('Markdown'),
+				}, () => new InteropService_Importer_Md()),
+
+				makeImportModule({
+					format: 'md_frontmatter',
+					fileExtensions: ['md', 'markdown', 'txt', 'html'],
+					sources: [FileSystemItem.File, FileSystemItem.Directory],
+					isNoteArchive: false, // Tells whether the file can contain multiple notes (eg. Enex or Jex format)
+					description: _('Markdown + Front Matter'),
+				}, () => new InteropService_Importer_Md_frontmatter()),
+
+				makeImportModule({
+					format: 'obsidian',
+					fileExtensions: ['md'],
+					sources: [FileSystemItem.Directory],
+					isNoteArchive: false,
+					description: _('Obsidian Vault'),
+					supportsMobile: true,
+				}, () => new InteropService_Importer_Obsidian()),
+
+				makeImportModule({
+					format: 'txt',
+					fileExtensions: ['txt'],
+					sources: [FileSystemItem.File, FileSystemItem.Directory],
+					isNoteArchive: false, // Tells whether the file can contain multiple notes (eg. Enex or Jex format)
+					description: _('Text document'),
+				}, () => new InteropService_Importer_Md()),
+
+				makeImportModule({
+					format: 'one',
+					fileExtensions: [
+						'zip',
+						'one',
+						'onepkg',
+					],
+					sources: [FileSystemItem.File],
+					isNoteArchive: true, // Tells whether the file can contain multiple notes (eg. Enex or Jex format)
+					description: _('OneNote Notebook'),
+				}, () => new InteropService_Importer_OneNote()),
+			];
+
+			const exportModules = [
+				makeExportModule({
+					format: ExportModuleOutputFormat.Jex,
+					fileExtensions: ['jex'],
+					target: FileSystemItem.File,
+					description: _('Joplin Export File'),
+				}, () => new InteropService_Exporter_Jex()),
+
+				makeExportModule({
+					format: ExportModuleOutputFormat.Raw,
+					target: FileSystemItem.Directory,
+					description: _('Joplin Export Directory'),
+				}, () => new InteropService_Exporter_Raw()),
+
+				makeExportModule({
+					format: ExportModuleOutputFormat.Markdown,
+					target: FileSystemItem.Directory,
+					description: _('Markdown'),
+				}, () => new InteropService_Exporter_Md()),
+
+				makeExportModule({
+					format: ExportModuleOutputFormat.MarkdownFrontMatter,
+					target: FileSystemItem.Directory,
+					description: _('Markdown + Front Matter'),
+				}, () => new InteropService_Exporter_Md_frontmatter()),
+
+				makeExportModule({
+					format: ExportModuleOutputFormat.Html,
+					fileExtensions: ['html', 'htm'],
+					target: FileSystemItem.File,
+					isNoteArchive: false,
+					description: _('HTML File'),
+					supportsMobile: false,
+				}, () => new InteropService_Exporter_Html()),
+
+				makeExportModule({
+					format: ExportModuleOutputFormat.Html,
+					target: FileSystemItem.Directory,
+					description: _('HTML Directory'),
+					supportsMobile: false,
+				}, () => new InteropService_Exporter_Html()),
+			];
+
+			this.defaultModules_ = (importModules as Module[]).concat(exportModules);
+		}
+
+		return this.defaultModules_.concat(this.userModules_);
+	}
+
+	public registerModule(module: Module) {
+		this.userModules_.push(module);
+		this.eventEmitter_.emit('modulesChanged');
+	}
+
+	public set xmlSerializer(xmlSerializer: XMLSerializer) {
+		this.xmlSerializer_ = xmlSerializer;
+	}
+
+	public get xmlSerializer() {
+		return this.xmlSerializer_;
+	}
+
+	public set domParser(domParser: DOMParser) {
+		this.domParser_ = domParser;
+	}
+
+	public get domParser() {
+		return this.domParser_;
+	}
+
+	// Find the module that matches the given type ("importer" or "exporter")
+	// and the given format. Some formats can have multiple associated importers
+	// or exporters, such as ENEX. In this case, the one marked as "isDefault"
+	// is returned. This is useful to auto-detect the module based on the format.
+	// For more precise matching, newModuleFromPath_ should be used.
+	public findModuleByFormat(type: ModuleType, format: string, target: FileSystemItem = null, outputFormat: ImportModuleOutputFormat = null) {
+		const modules = this.modules();
+		const matches = [];
+
+		const isMobile = shim.mobilePlatform() !== '';
+		for (let i = 0; i < modules.length; i++) {
+			const m = modules[i];
+
+			if (!m.supportsMobile && isMobile) {
+				continue;
+			}
+
+			if (m.format === format && m.type === type) {
+				if (!target && !outputFormat) {
+					matches.push(m);
+				} else if (
+					m.type === ModuleType.Exporter && target && target === m.target
+				) {
+					matches.push(m);
+				} else if (
+					m.type === ModuleType.Importer && outputFormat && outputFormat === m.outputFormat
+				) {
+					matches.push(m);
+				}
+			}
+		}
+
+		const output = matches.find(m => !!m.isDefault);
+		if (output) return output;
+
+		return matches.length ? matches[0] : null;
+	}
+
+	// NOTE TO FUTURE SELF: It might make sense to simply move all the existing
+	// formatters to the `newModuleFromPath_` approach, so that there's only one way
+	// to do this mapping. This isn't a priority right now (per the convo in:
+	// https://github.com/laurent22/joplin/pull/1795#discussion_r322379121) but
+	// we can do it if it ever becomes necessary.
+	private newModuleByFormat_(type: ModuleType, format: string, outputFormat: ImportModuleOutputFormat = ImportModuleOutputFormat.Markdown) {
+		const moduleMetadata = this.findModuleByFormat(type, format, null, outputFormat);
+		if (!moduleMetadata) throw new Error(_('Cannot load "%s" module for format "%s" and output "%s"', type, format, outputFormat));
+
+		return moduleMetadata.factory();
+	}
+
+	// The existing `newModuleByFormat_` fn would load by the input format. This
+	// was fine when there was a 1-1 mapping of input formats to output formats,
+	// but now that we have 2 possible outputs for an `enex` input, we need to be
+	// explicit with which importer we want to use.
+	//
+	// https://github.com/laurent22/joplin/pull/1795#pullrequestreview-281574417
+	private newModuleFromPath_(type: ModuleType, options: ExportOptions&ImportOptions) {
+		const moduleMetadata = this.findModuleByFormat(type, options.format, options.target);
+		if (!moduleMetadata) throw new Error(_('Cannot load "%s" module for format "%s" and target "%s"', type, options.format, options.target));
+
+		return moduleMetadata.factory(options);
+	}
+
+	private moduleByFileExtension_(type: ModuleType, ext: string) {
+		ext = ext.toLowerCase();
+
+		const modules = this.modules();
+
+		for (let i = 0; i < modules.length; i++) {
+			const m = modules[i];
+			if (type !== m.type) continue;
+			if (m.fileExtensions.includes(ext)) return m;
+		}
+
+		return null;
+	}
+
+	public async import(options: ImportOptions): Promise<ImportExportResult> {
+		if (!(await shim.fsDriver().exists(options.path))) throw new Error(_('Cannot find "%s".', options.path));
+
+		options = {
+			format: 'auto',
+			destinationFolderId: null,
+			destinationFolder: null,
+			xmlSerializer: this.xmlSerializer,
+			domParser: this.domParser,
+			...options,
+		};
+
+		if (options.format === 'auto') {
+			const module = this.moduleByFileExtension_(ModuleType.Importer, fileExtension(options.path));
+			if (!module) throw new Error(_('Please specify import format for %s', options.path));
+			options.format = module.format;
+		}
+
+		if (options.destinationFolderId) {
+			const folder = await Folder.load(options.destinationFolderId);
+			if (!folder) throw new Error(_('Cannot find "%s".', options.destinationFolderId));
+			options.destinationFolder = folder;
+		}
+
+		let result: ImportExportResult = { warnings: [] };
+
+		const importer = this.newModuleByFormat_(ModuleType.Importer, options.format, options.outputFormat);
+
+		if (!(importer instanceof InteropService_Importer_Base)) {
+			throw new Error('Resolved importer is not an importer');
+		}
+
+		await importer.init(options.path, options);
+		result = await importer.exec(result);
+
+		return result;
+	}
+
+	private normalizeItemForExport<T extends Record<string, unknown>>(_itemType: ModelType, item: T): T {
+		const override: Partial<{ is_shared: number; share_id: string }> = {};
+		if ('is_shared' in item) override.is_shared = 0;
+		if ('share_id' in item) override.share_id = '';
+
+		if (Object.keys(override).length) {
+			return {
+				...item,
+				...override,
+			};
+		} else {
+			return item;
+		}
+	}
+
+	private async decryptedNoteForExport_(noteId: string, noteLockKey: DecryptedNoteLockKey): Promise<NoteEntity|null> {
+		if (!noteLockKey) return null;
+		try {
+			const decrypted: NoteLockNoteEntity = await Note.load(noteId, { useNoteLock: true, noteLockKey });
+			// Without the marker the load was not gated, as when the feature is turned off midway, so the body is still ciphertext.
+			if (!decrypted.isDecrypted) return null;
+			delete decrypted.isDecrypted;
+			return { ...decrypted, is_locked: 0, extracted_resource_ids: '' };
+		} catch {
+			return null;
+		}
+	}
+
+	public async export(options: ExportOptions): Promise<ImportExportResult> {
+		options = {
+			format: ExportModuleOutputFormat.Jex,
+			...options,
+		};
+
+		const exportPath = options.path ? options.path : null;
+		let sourceFolderIds = options.sourceFolderIds ? options.sourceFolderIds : [];
+		const sourceNoteIds = options.sourceNoteIds ? options.sourceNoteIds : [];
+		const result: ImportExportResult = { warnings: [] };
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any -- itemsToExport is consumed by exporter.prepareForProcessingItemType (typed BaseItemEntity[]); the actual values are { type, itemOrId } structurally — narrowing here would force changing the exporter signature
+		const itemsToExport: any[] = [];
+
+		options.onProgress?.(ExportProgressState.QueuingItems, null);
+		let totalItemsToProcess = 0;
+
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any -- See itemsToExport above
+		const queueExportItem = (itemType: number, itemOrId: any) => {
+			totalItemsToProcess ++;
+			itemsToExport.push({
+				type: itemType,
+				itemOrId: itemOrId,
+			});
+		};
+
+		const exportedNoteIds = [];
+		let resourceIds: string[] = [];
+		// Backups keep locked notes encrypted; other formats export them decrypted only when decryptLockedNotes is set.
+		const keepsLockedNotes = options.format === ExportModuleOutputFormat.Raw || options.format === ExportModuleOutputFormat.Jex;
+		let lockedNotesSkipped = 0;
+		// Read once, so enabling the feature partway through cannot leave the locked notes queued before it as ciphertext.
+		// Turning it off partway is caught by the gated load in decryptedNoteForExport_.
+		const noteLockEnabled = isNoteLockEnabled();
+		// Captured at the start, so locking the session partway through does not skip the remaining locked notes.
+		const noteLockKey = NoteLockSession.instance().isUnlocked() ? NoteLockSession.instance().decryptedKey() : null;
+
+		// Recursively get all the folders that have valid parents
+		const folderIds = await Folder.childrenIds('');
+
+		if (options.includeConflicts) folderIds.push(Folder.conflictFolderId());
+
+		let fullSourceFolderIds = sourceFolderIds.slice();
+		for (let i = 0; i < sourceFolderIds.length; i++) {
+			const id = sourceFolderIds[i];
+			const childrenIds = await Folder.childrenIds(id);
+			fullSourceFolderIds = fullSourceFolderIds.concat(childrenIds);
+		}
+		sourceFolderIds = fullSourceFolderIds;
+
+		for (let folderIndex = 0; folderIndex < folderIds.length; folderIndex++) {
+			const folderId = folderIds[folderIndex];
+			if (sourceFolderIds.length && sourceFolderIds.indexOf(folderId) < 0) continue;
+
+			if (!sourceNoteIds.length) await queueExportItem(BaseModel.TYPE_FOLDER, folderId);
+
+			const noteIds = await Folder.noteIds(folderId, { includeConflicts: !!options.includeConflicts });
+
+			for (let noteIndex = 0; noteIndex < noteIds.length; noteIndex++) {
+				const noteId = noteIds[noteIndex];
+				if (sourceNoteIds.length && sourceNoteIds.indexOf(noteId) < 0) continue;
+				let note = await Note.load(noteId);
+				if (noteLockEnabled && NoteLockNote.isLocked(note) && !keepsLockedNotes) {
+					if (options.skipLockedNotes) continue;
+					const decrypted = options.decryptLockedNotes ? await this.decryptedNoteForExport_(noteId, noteLockKey) : null;
+					if (!decrypted) {
+						lockedNotesSkipped++;
+						continue;
+					}
+					note = decrypted;
+				}
+				await queueExportItem(BaseModel.TYPE_NOTE, note);
+				exportedNoteIds.push(noteId);
+
+				// A locked note's body is ciphertext, so its resource ids come from the extracted list.
+				const rids = noteLockEnabled && NoteLockNote.isLocked(note)
+					? Note.unserializeExtractedResourceIds(note.extracted_resource_ids)
+					: await Note.linkedResourceIds(note.body);
+				resourceIds = resourceIds.concat(rids);
+			}
+		}
+
+		// Nothing would be written, so a single-file export would otherwise fail on the file it never wrote.
+		if (lockedNotesSkipped && !exportedNoteIds.length) {
+			throw new Error(_n('%d locked note could not be unlocked and was not exported.', '%d locked notes could not be unlocked and were not exported.', lockedNotesSkipped, lockedNotesSkipped));
+		}
+
+		resourceIds = ArrayUtils.unique(resourceIds);
+
+		for (let i = 0; i < resourceIds.length; i++) {
+			await queueExportItem(BaseModel.TYPE_RESOURCE, resourceIds[i]);
+		}
+
+		const noteTags = await NoteTag.all();
+
+		const exportedTagIds = [];
+
+		for (let i = 0; i < noteTags.length; i++) {
+			const noteTag = noteTags[i];
+			if (exportedNoteIds.indexOf(noteTag.note_id) < 0) continue;
+			await queueExportItem(BaseModel.TYPE_NOTE_TAG, noteTag.id);
+			exportedTagIds.push(noteTag.tag_id);
+		}
+
+		for (let i = 0; i < exportedTagIds.length; i++) {
+			await queueExportItem(BaseModel.TYPE_TAG, exportedTagIds[i]);
+		}
+
+		const exporter = this.newModuleFromPath_(ModuleType.Exporter, options);
+		if (!(exporter instanceof InteropService_Exporter_Base)) {
+			throw new Error('Resolved exporter is not an exporter');
+		}
+
+		await exporter.init(exportPath, options);
+
+		const typeOrder = [BaseModel.TYPE_FOLDER, BaseModel.TYPE_RESOURCE, BaseModel.TYPE_NOTE, BaseModel.TYPE_TAG, BaseModel.TYPE_NOTE_TAG];
+		const context: { resourcePaths: Record<string, string>; warnings: string[]; destResourcePaths?: Record<string, string>; notePaths?: Record<string, string> } = {
+			resourcePaths: {},
+			warnings: result.warnings,
+		};
+		exporter.updateContext(context);
+
+		// Prepare to process each type before starting any
+		// This will allow exporters to operate on the full context
+		for (let typeOrderIndex = 0; typeOrderIndex < typeOrder.length; typeOrderIndex++) {
+			const type = typeOrder[typeOrderIndex];
+
+			await exporter.prepareForProcessingItemType(type, itemsToExport);
+		}
+
+		let itemsProcessed = 0;
+		for (let typeOrderIndex = 0; typeOrderIndex < typeOrder.length; typeOrderIndex++) {
+			const type = typeOrder[typeOrderIndex];
+
+			for (let i = 0; i < itemsToExport.length; i++) {
+				const itemType = itemsToExport[i].type;
+
+				if (itemType !== type) continue;
+
+				const ItemClass = BaseItem.getClassByItemType(itemType);
+				const itemOrId = itemsToExport[i].itemOrId;
+				const rawItem = typeof itemOrId === 'object' ? itemOrId : await ItemClass.load(itemOrId);
+
+				if (!rawItem) {
+					if (itemType === BaseModel.TYPE_RESOURCE) {
+						result.warnings.push(sprintf('A resource that does not exist is referenced in a note. The resource was skipped. Resource ID: %s', itemOrId));
+					} else {
+						result.warnings.push(sprintf('Cannot find item with type "%s" and ID %s. Item was skipped.', ItemClass.tableName(), JSON.stringify(itemOrId)));
+					}
+					continue;
+				}
+
+				const item = this.normalizeItemForExport(itemType, rawItem);
+
+				if (item.encryption_applied || item.encryption_blob_encrypted) {
+					result.warnings.push(sprintf('This item is currently encrypted: %s "%s" (%s) and was not exported. You may wait for it to be decrypted and try again.', BaseModel.modelTypeToName(itemType), item.title ? item.title : item.id, item.id));
+					continue;
+				}
+
+				try {
+					if (itemType === BaseModel.TYPE_RESOURCE) {
+						const resourcePath = Resource.fullPath(item);
+						context.resourcePaths[item.id] = resourcePath;
+						exporter.updateContext(context);
+						await exporter.processResource(item, resourcePath);
+					}
+
+					await exporter.processItem(itemType, item);
+				} catch (error) {
+					console.error(error);
+					result.warnings.push(error.message);
+				}
+
+				itemsProcessed++;
+				options.onProgress?.(ExportProgressState.Exporting, itemsProcessed / totalItemsToProcess);
+			}
+		}
+
+		options.onProgress?.(ExportProgressState.Closing, null);
+		await exporter.close();
+
+		if (lockedNotesSkipped) result.warnings.push(`${lockedNotesSkipped} locked note(s) could not be unlocked and were not exported`);
+
+		return result;
+	}
+}