mynode/packages/lib/services/ExternalEditWatcher.ts

import Logger from '@joplin/utils/Logger';
import Setting from '../models/Setting';
import shim from '../shim';
import { basename, toSystemSlashes } from '../path-utils';
import time from '../time';
import { NoteEntity } from './database/types';
import Note from '../models/Note';
import { openFileWithExternalEditor } from './ExternalEditWatcher/utils';
import AsyncActionQueue from '../AsyncActionQueue';
import { EventEmitter } from 'events';
import * as chokidar from 'chokidar';
const md5 = require('md5');
const { ErrorNotFound } = require('./rest/utils/errors');

interface ChangeEventContext {
	path: string;
}

type DispatchFn = (action: { type: string; [key: string]: unknown })=> void;
// Bridge() returns the desktop bridge with methods openItem/openExternal/etc. used by the external-editor helper.
// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Concrete bridge type lives in app-desktop; lib references it structurally
type BridgeFn = ()=> any;

const hashNoteContent = (content: string) => md5(content);
type ItemId = string;

export default class ExternalEditWatcher {

	private dispatch: DispatchFn;
	private bridge_: BridgeFn;
	private logger_: Logger = new Logger();
	// eslint-disable-next-line @typescript-eslint/no-explicit-any -- chokidar typings vary across platforms; we use a small subset (watch/on/unwatch/close) treated structurally
	private watcher_: any = null;
	private changeEventQueue_: AsyncActionQueue<ChangeEventContext>;
	private eventEmitter_: EventEmitter = new EventEmitter();
	private skipNextChangeEvent_: Map<ItemId, boolean> = new Map();
	private lastNoteContentHash_: Map<ItemId, string> = new Map();
	// eslint-disable-next-line @typescript-eslint/no-explicit-any -- See watcher_ above
	private chokidar_: any = chokidar;
	private watcherReady_: Promise<void> = null;

	private static instance_: ExternalEditWatcher;

	public static instance() {
		if (this.instance_) return this.instance_;
		this.instance_ = new ExternalEditWatcher();
		return this.instance_;
	}

	public initialize(bridge: BridgeFn, dispatch: DispatchFn) {
		this.bridge_ = bridge;
		this.dispatch = dispatch;

		// Chokidar skips repeated events for the same file.
		// As a result, it may be possible for something similar to the following to happen:
		// 1. A user makes a change to a file in an external editor.
		// 2. The external editor truncates the file to just before the change.
		// 3. Joplin receives an event from Chokidar and reads the truncated file.
		// 4. The external editor writes the remainder of the file (with the user's change).
		// 5. The change event is ignored (throttled by Chokidar) since it happened within
		//    50 ms of the change from step 2.
		// 6. The note remains truncated in Joplin.
		//
		// To avoid this, we use an AsyncActionQueue to delay processing change events received
		// from Chokidar.
		// The timeout must be at least the throttling delay used internally by chokidar.
		// With Chokidar 3.6.0, this throttling delay is 50ms.
		// https://github.com/laurent22/joplin/issues/14954
		const timeout = 55;
		this.changeEventQueue_ = new AsyncActionQueue<ChangeEventContext>(timeout);
		this.changeEventQueue_.setCanSkipTaskHandler((task1, task2) => {
			// Only skip change events for the same path
			const context1 = task1.context;
			const context2 = task2.context;
			return context1.path === context2.path;
		});
	}

	public externalApi() {
		const loadNote = async (noteId: string) => {
			const note = await Note.load(noteId);
			if (!note) throw new ErrorNotFound(`No such note: ${noteId}`);
			return note;
		};

		return {
			openAndWatch: async (args: { noteId: string }) => {
				const note = await loadNote(args.noteId);
				return this.openAndWatch(note);
			},
			stopWatching: async (args: { noteId: string }) => {
				return this.stopWatching(args.noteId);
			},
			noteIsWatched: async (args: { noteId: string }) => {
				const note = await loadNote(args.noteId);
				return this.noteIsWatched(note);
			},
		};
	}

	public tempDir() {
		return Setting.value('profileDir');
	}

	// eslint-disable-next-line @typescript-eslint/no-explicit-any -- EventEmitter events carry heterogeneous payloads by name; per-event typing would need a string-literal map across all callers
	public on(eventName: string, callback: (...args: any[])=> void) {
		return this.eventEmitter_.on(eventName, callback);
	}

	// eslint-disable-next-line @typescript-eslint/no-explicit-any -- See on() above
	public off(eventName: string, callback: (...args: any[])=> void) {
		return this.eventEmitter_.removeListener(eventName, callback);
	}

	public setLogger(l: Logger) {
		this.logger_ = l;
	}

	public logger() {
		return this.logger_;
	}

	public watch(fileToWatch: string) {
		if (!this.chokidar_) return;

		if (!this.watcher_) {
			this.watcher_ = this.chokidar_.watch(fileToWatch, {
				useFsEvents: false,
			});

			this.watcherReady_ = new Promise<void>(resolve => {
				this.watcher_.on('ready', () => resolve());
			});

			this.watcher_.on('all', async (event: string, path: string) => {
				this.logger().debug(`ExternalEditWatcher: Event: ${event}: ${path}`);

				if (event === 'unlink') {
					// File are unwatched in the stopWatching functions below. When we receive an unlink event
					// here it might be that the file is quickly moved to a different location and replaced by
					// another file with the same name, as it happens with emacs. So because of this
					// we keep watching anyway.
					// See: https://github.com/laurent22/joplin/issues/710#issuecomment-420997167
					// this.watcher_.unwatch(path);
				} else if (event === 'change') {
					const id = this.noteFilePathToId_(path);

					if (!this.skipNextChangeEvent_.has(id)) {
						this.changeEventQueue_.push(() => this.onNoteChange_(path), { path });
					} else {
						this.logger().debug('ExternalEditWatcher: Skipping this event.');
						this.skipNextChangeEvent_.delete(id);
					}
				} else if (event === 'error') {
					this.logger().error('ExternalEditWatcher: error');
				}
			});
		} else {
			this.watcher_.add(fileToWatch);
		}

		return this.watcher_;
	}

	private async onNoteChange_(path: string) {
		const id = this.noteFilePathToId_(path);
		const note = await Note.load(id);

		if (!note) {
			this.logger().warn(`ExternalEditWatcher: Watched note has been deleted: ${id}`);
			void this.stopWatching(id);
			return;
		}

		let noteContent: string;
		try {
			noteContent = await shim.fsDriver().readFile(path, 'utf-8');
		} catch (error) {
			if (error.code === 'ENOENT') {
				this.logger().warn(`ExternalEditWatcher: Watched file no longer exists: ${path}`);
				void this.stopWatching(id);
				return;
			}
			throw error;
		}

		// In some very rare cases, the "change" event is going to be emitted but the file will be empty.
		// This is likely to be the editor that first clears the file, then writes the content to it, so if
		// the file content is read very quickly after the change event, we'll get empty content.
		// Usually, re-reading the content again will fix the issue and give back the file content.
		// To replicate on Windows: associate Typora as external editor, and leave Ctrl+S pressed -
		// it will keep on saving very fast and the bug should happen at some point.
		// Below we re-read the file multiple times until we get the content, but in my tests it always
		// work in the first try anyway. The loop is just for extra safety.
		// https://github.com/laurent22/joplin/issues/1854
		if (!noteContent) {
			this.logger().warn(`ExternalEditWatcher: Watched note is empty - this is likely to be a bug and re-reading the note should fix it. Trying again... ${id}`);

			for (let i = 0; i < 10; i++) {
				noteContent = await shim.fsDriver().readFile(path, 'utf-8');
				if (noteContent) {
					this.logger().info(`ExternalEditWatcher: Note is now readable: ${id}`);
					break;
				}
				await time.msleep(100);
			}

			if (!noteContent) this.logger().warn(`ExternalEditWatcher: Could not re-read note - user might have purposely deleted note content: ${id}`);
		}

		const contentHash = hashNoteContent(noteContent);
		if (contentHash === this.lastNoteContentHash_.get(id)) {
			this.logger().info('ExternalEditWatcher: No changes since last write');
			return;
		}

		this.logger().debug('ExternalEditWatcher: Updating note object.');

		const updatedNote = await Note.unserializeForEdit(noteContent);
		updatedNote.id = id;
		updatedNote.parent_id = note.parent_id;
		await Note.save(updatedNote);
		this.eventEmitter_.emit('noteChange', { id: updatedNote.id, note: updatedNote });

		this.lastNoteContentHash_.set(id, contentHash);
	}

	private noteIdToFilePath_(noteId: string) {
		return `${this.tempDir()}/edit-${noteId}.md`;
	}

	private noteFilePathToId_(path: string) {
		const pathParts = toSystemSlashes(path, 'linux').split('/');
		if (!pathParts.length) throw new Error(`Invalid path: ${path}`);
		const fileName = pathParts[pathParts.length - 1];
		const fileNameParts = fileName.split('.');
		fileNameParts.pop();
		const idParts = fileNameParts[0].split('-');
		return idParts[1];
	}

	public watchedFiles() {
		if (!this.watcher_) return [];

		const output = [];
		const watchedPaths = this.watcher_.getWatched();

		for (const dirName in watchedPaths) {
			if (!watchedPaths.hasOwnProperty(dirName)) continue;

			for (let i = 0; i < watchedPaths[dirName].length; i++) {
				const f = watchedPaths[dirName][i];
				output.push(`${this.tempDir()}/${f}`);
			}
		}

		return output;
	}

	public noteIsWatched(note: NoteEntity) {
		if (!this.watcher_) return false;

		const noteFilename = basename(this.noteIdToFilePath_(note.id));

		const watchedPaths = this.watcher_.getWatched();

		for (const dirName in watchedPaths) {
			if (!watchedPaths.hasOwnProperty(dirName)) continue;

			for (let i = 0; i < watchedPaths[dirName].length; i++) {
				const f = watchedPaths[dirName][i];
				if (f === noteFilename) return true;
			}
		}

		return false;
	}

	public async openAndWatch(note: NoteEntity) {
		if (!note || !note.id) {
			this.logger().warn('ExternalEditWatcher: Cannot open note: ', note);
			return;
		}

		const filePath = await this.writeNoteToFile_(note);
		if (!filePath) return;
		this.watch(filePath);

		// A save made before Chokidar's initial scan completes is folded into its "add" event,
		// which we ignore, so the change would be lost.
		await this.watcherReady_;

		await openFileWithExternalEditor(filePath, this.bridge_());

		this.dispatch({
			type: 'NOTE_FILE_WATCHER_ADD',
			id: note.id,
		});

		this.logger().info(`ExternalEditWatcher: Started watching ${filePath}`);
	}

	// Without a flush, a pending change finds the file removed below and is dropped.
	public async stopWatching(noteId: string, flushPendingChanges = true) {
		if (!noteId) return;

		if (flushPendingChanges) await this.changeEventQueue_.processAllNow();

		const filePath = this.noteIdToFilePath_(noteId);
		if (this.watcher_) this.watcher_.unwatch(filePath);
		await shim.fsDriver().remove(filePath);

		this.lastNoteContentHash_.delete(noteId);
		this.skipNextChangeEvent_.delete(noteId);
		this.dispatch({
			type: 'NOTE_FILE_WATCHER_REMOVE',
			id: noteId,
		});
		this.logger().info(`ExternalEditWatcher: Stopped watching ${filePath}`);
	}

	public async stopWatchingAll() {
		await this.changeEventQueue_.processAllNow();

		const filePaths = this.watchedFiles();
		for (let i = 0; i < filePaths.length; i++) {
			await shim.fsDriver().remove(filePaths[i]);
		}

		if (this.watcher_) this.watcher_.close();
		this.watcher_ = null;
		this.watcherReady_ = null;
		this.skipNextChangeEvent_.clear();
		this.lastNoteContentHash_.clear();
		this.logger().info('ExternalEditWatcher: Stopped watching all files');
		this.dispatch({
			type: 'NOTE_FILE_WATCHER_CLEAR',
		});
	}

	public async updateNoteFile(note: NoteEntity) {
		if (!this.noteIsWatched(note)) return;

		if (!note || !note.id) {
			this.logger().warn('ExternalEditWatcher: Cannot update note file: ', note);
			return;
		}

		this.logger().debug(`ExternalEditWatcher: Update note file: ${note.id}`);

		// When the note file is updated programmatically, we skip the next change event to
		// avoid update loops. We only want to listen to file changes made by the user.
		this.skipNextChangeEvent_.set(note.id, true);

		await this.writeNoteToFile_(note);
	}

	private async writeNoteToFile_(note: NoteEntity) {
		if (!note || !note.id) {
			this.logger().warn('ExternalEditWatcher: Cannot update note file: ', note);
			return null;
		}

		const filePath = this.noteIdToFilePath_(note.id);
		const noteContent = await Note.serializeForEdit(note);

		await shim.fsDriver().writeFile(filePath, noteContent, 'utf-8');
		this.lastNoteContentHash_.set(note.id, hashNoteContent(noteContent));

		return filePath;
	}
}