mynode/packages/lib/services/RevisionService.ts

import ItemChange from '../models/ItemChange';
import Note from '../models/Note';
import Folder from '../models/Folder';
import Setting from '../models/Setting';
import Revision from '../models/Revision';
import BaseModel, { DeleteOptions } from '../BaseModel';
import ItemChangeUtils from './ItemChangeUtils';
import shim from '../shim';
import BaseService from './BaseService';
import { _ } from '../locale';
import { ItemChangeEntity, NoteEntity, RevisionEntity } from './database/types';
import Logger from '@joplin/utils/Logger';
import isNoteLockEnabled from './noteLock/isNoteLockEnabled';
import NoteLockNote from './noteLock/NoteLockNote';
import { MarkupLanguage } from '../../renderer';
import { substrWithEllipsis } from '../string-utils';
const { sprintf } = require('sprintf-js');
import { wrapError } from '../errorUtils';

const logger = Logger.create('RevisionService');

export default class RevisionService extends BaseService {

	public static instance_: RevisionService;

	private changedSinceCollectionCache_: Set<string> = new Set();
	private maintenanceCalls_: boolean[] = [];
	private maintenanceTimer1_: ReturnType<typeof shim.setTimeout> = null;
	private maintenanceTimer2_: ReturnType<typeof shim.setInterval> = null;
	private isCollecting_ = false;
	public isRunningInBackground_ = false;

	public static instance() {
		if (this.instance_) return this.instance_;
		this.instance_ = new RevisionService();
		return this.instance_;
	}

	// An "old note" is a note which is without any history, or has not had a revision created since the defined oldNoteInterval.
	// Originally this concept was introduced to define a note that was created before the revision service existed, but actually
	// this also applies to any note that is history-less, which is also the case for notes which have been imported, or notes which
	// had their note history deleted by the user or by the revision cleaner.
	// These notes are without revisions so the first time they are modified, a copy of the original note is saved. The goal is to
	// have at least one revision in case the note is deleted or modified as a result of a bug or user mistake.
	// Also, because of the fact that the revision collection will not always create a revision for the latest change (due to the
	// intervalBetweenRevisions restriction), it is beneficial to determine an old note based on an interval rather than solely on
	// the existence of at least 1 revision for a note. Therefore it is beneficial for the old note concept to include any note
	// which has not been changed since the defined oldNoteInterval.
	public oldNoteCutOffDate_() {
		return Date.now() - Setting.value('revisionService.oldNoteInterval');
	}

	public changedSinceCollection(noteId: string) {
		if (this.changedSinceCollectionCache_.has(noteId)) return true;

		this.changedSinceCollectionCache_.add(noteId);

		return false;
	}

	public removeChangedSinceCollection(noteId: string) {
		this.changedSinceCollectionCache_.delete(noteId);
	}

	private noteMetadata_(note: NoteEntity) {
		const excludedFields = ['type_', 'title', 'body', 'created_time', 'updated_time', 'encryption_applied', 'encryption_cipher_text', 'is_conflict', 'user_data'];
		const md: Record<string, unknown> = {};
		const noteRecord = note as unknown as Record<string, unknown>;
		for (const k in note) {
			if (excludedFields.indexOf(k) >= 0) continue;
			md[k] = noteRecord[k];
		}

		if (note.user_updated_time === note.updated_time) delete md.user_updated_time;
		if (note.user_created_time === note.created_time) delete md.user_created_time;

		return md;
	}

	public async createNoteRevision_(note: NoteEntity, parentRevId: string = null, bypassInterval = false): Promise<RevisionEntity> {
		try {
			const parentRev = parentRevId ? await Revision.load(parentRevId) : await Revision.latestRevision(BaseModel.TYPE_NOTE, note.id);

			const output: RevisionEntity = {
				parent_id: '',
				item_type: BaseModel.TYPE_NOTE,
				item_id: note.id,
				item_updated_time: note.updated_time,
				// Carried with the flag off too, so the revision still marks its body as ciphertext.
				is_locked: note.is_locked ? 1 : 0,
			};

			const noteMd = this.noteMetadata_(note);
			const noteTitle = note.title ? note.title : '';
			const noteBody = note.body ? note.body : '';

			if (isNoteLockEnabled() && NoteLockNote.isLocked(note)) {
				// A locked note body is ciphertext, so diffing it against other revisions would produce
				// large meaningless patches. Each revision is standalone instead: no parent, full contents.
				if (parentRev && !bypassInterval && Date.now() - parentRev.updated_time < Setting.value('revisionService.intervalBetweenRevisions')) return null;
				output.title_diff = Revision.createTextPatch('', noteTitle);
				output.body_diff = Revision.createTextPatch('', noteBody);
				output.metadata_diff = Revision.createObjectPatch({}, noteMd);
			} else if (!parentRev || !!parentRev.is_locked !== !!note.is_locked) {
				// A chain never crosses a lock state change, so clearing plaintext history cannot orphan a locked revision.
				output.title_diff = Revision.createTextPatch('', noteTitle);
				output.body_diff = Revision.createTextPatch('', noteBody);
				output.metadata_diff = Revision.createObjectPatch({}, noteMd);
			} else {
				if (!bypassInterval && Date.now() - parentRev.updated_time < Setting.value('revisionService.intervalBetweenRevisions')) return null;

				const merged = await Revision.mergeDiffs(parentRev);
				output.parent_id = parentRev.id;
				output.title_diff = Revision.createTextPatch(merged.title, noteTitle);
				output.body_diff = Revision.createTextPatch(merged.body, noteBody);
				output.metadata_diff = Revision.createObjectPatch(merged.metadata, noteMd);
			}

			if (Revision.isEmptyRevision(output)) return null;

			return Revision.save(output);
		} catch (error) {
			const newError = wrapError(`Could not create revision for note: ${note.id}`, error);
			throw newError;
		}
	}

	public async collectRevisions() {
		if (this.isCollecting_) return;

		this.isCollecting_ = true;

		await ItemChange.waitForAllSaved();

		const doneNoteIds = [];

		try {
			while (true) {
				// See synchronizer test units to see why changes coming
				// from sync are skipped.
				const changes: ItemChangeEntity[] = await ItemChange.modelSelectAll(
					`
					SELECT id, item_id, type, before_change_item
					FROM item_changes
					WHERE item_type = ?
					AND source != ?
					AND source != ?
					AND id > ?
					ORDER BY id ASC
					LIMIT 10
				`,
					[BaseModel.TYPE_NOTE, ItemChange.SOURCE_SYNC, ItemChange.SOURCE_DECRYPTION, Setting.value('revisionService.lastProcessedChangeId')],
				);

				if (!changes.length) break;

				const noteIds = changes.map((a) => a.item_id);
				const notes = await Note.modelSelectAll(`SELECT * FROM notes WHERE is_conflict = 0 AND encryption_applied = 0 AND id IN (${
					Note.escapeIdsForSql(noteIds)
				})`);

				for (let i = 0; i < changes.length; i++) {
					const change = changes[i];
					const noteId = change.item_id;

					try {
						if (change.type === ItemChange.TYPE_UPDATE && doneNoteIds.indexOf(noteId) < 0) {
							const note = BaseModel.byId(notes, noteId);
							const oldNote = change.before_change_item ? JSON.parse(change.before_change_item) : null;

							if (note) {
								let oldNoteSaved = false;

								if (oldNote && oldNote.updated_time < this.oldNoteCutOffDate_()) {
									// This is where we save the original version of this old note
									// We need to use the more recent timestamp, because if the last update was a long time ago, the revision could get immediately removed by the cleaner
									// We also want to avoid creating 2 revisions with exactly the same timestamp, so deduct 1 ms from the timestamp on the old revision to avoid this
									oldNote.updated_time = note.updated_time - 1;
									oldNote.user_updated_time = oldNote.updated_time;
									const rev = await this.createNoteRevision_(oldNote);
									if (rev) {
										oldNoteSaved = true;
										logger.debug(sprintf('collectRevisions: Saved revision %s (old note)', rev.id));
									}
								}

								const rev = await this.createNoteRevision_(note, null, oldNoteSaved);
								if (rev) logger.debug(sprintf('collectRevisions: Saved revision %s (Last rev was more than %d ms ago)', rev.id, Setting.value('revisionService.intervalBetweenRevisions')));
								doneNoteIds.push(noteId);

								this.changedSinceCollectionCache_.delete(noteId);
							}
						}

						if (change.type === ItemChange.TYPE_DELETE && !!change.before_change_item) {
							const note = JSON.parse(change.before_change_item);
							const revExists = await Revision.revisionExists(BaseModel.TYPE_NOTE, note.id, note.updated_time);
							if (!revExists) {
								const rev = await this.createNoteRevision_(note);
								if (rev) logger.debug(sprintf('collectRevisions: Saved revision %s (for deleted note)', rev.id));
							}
							doneNoteIds.push(noteId);
						}
					} catch (error) {
						if (error.code === 'revision_encrypted') {
							throw error;
						} else {
							// If any revision creation fails, we continue
							// processing the other changes. It seems a rare bug
							// in diff-match-patch can cause the creation of
							// revisions to fail in some case. It should be rare
							// and it's best to continue processing the other
							// changes. The alternative would be to stop here
							// and fix the bug, but in the meantime revisions
							// will no longer be generated.

							// The drawback is that once a change has been
							// skipped it will never be processed again because
							// the error will be in the past (before
							// revisionService.lastProcessedChangeId)
							//
							// https://github.com/laurent22/joplin/issues/5531
							logger.error(`collectRevisions: Processing one of the changes for note ${noteId} failed. Other changes will still be processed. Error was: `, error);
							logger.error('collectRevisions: Change was:', change);
						}
					}

					Setting.setValue('revisionService.lastProcessedChangeId', change.id);
				}
			}
		} catch (error) {
			if (error.code === 'revision_encrypted') {
				// One or more revisions are encrypted - stop processing for now
				// and these revisions will be processed next time the revision
				// collector runs.
				logger.info('collectRevisions: One or more revision was encrypted. Processing was stopped but will resume later when the revision is decrypted.', error);
			} else {
				// This should not happen anymore because we handle the error in
				// the loop above.
				logger.error('collectRevisions:', error);
			}
		}

		await Setting.saveAll();
		await ItemChangeUtils.deleteProcessedChanges();

		this.isCollecting_ = false;

		logger.info(`collectRevisions: Created revisions for ${doneNoteIds.length} notes`);
	}

	public async deleteOldRevisions(ttl: number) {
		return Revision.deleteOldRevisions(ttl);
	}

	public async revisionNote(revisions: RevisionEntity[], index: number) {
		if (index < 0 || index >= revisions.length) throw new Error(`Invalid revision index: ${index}`);

		const rev = revisions[index];
		const merged = await Revision.mergeDiffs(rev, revisions);

		const output: NoteEntity = {
			title: merged.title,
			body: merged.body,
			...merged.metadata,
		};
		output.updated_time = output.user_updated_time;
		output.created_time = output.user_created_time;
		(output as NoteEntity & { type_?: number }).type_ = BaseModel.TYPE_NOTE;
		if (!('markup_language' in output)) output.markup_language = MarkupLanguage.Markdown;

		return output;
	}

	public restoreFolderTitle() {
		return _('Restored Notes');
	}

	public async restoreFolder() {
		let folder = await Folder.loadByFields({
			title: this.restoreFolderTitle(),
			deleted_time: 0,
		});
		if (!folder) {
			folder = await Folder.save({ title: this.restoreFolderTitle() });
		}
		return folder;
	}

	// reverseRevIndex = 0 means restoring the latest version. reverseRevIndex =
	// 1 means the version before that, etc.
	public async restoreNoteById(noteId: string, reverseRevIndex: number): Promise<NoteEntity> {
		const revisions = await Revision.allByType(BaseModel.TYPE_NOTE, noteId);
		if (!revisions.length) throw new Error(`No revision for note "${noteId}"`);

		const revIndex = revisions.length - 1 - reverseRevIndex;

		const note = await this.revisionNote(revisions, revIndex);
		return this.importRevisionNote(note);
	}

	public restoreSuccessMessage(note: NoteEntity): string {
		return _('The note "%s" has been successfully restored to the notebook "%s".', substrWithEllipsis(note.title, 0, 32), this.restoreFolderTitle());
	}

	public async importRevisionNote(note: NoteEntity): Promise<NoteEntity> {
		const toImport = { ...note };
		delete toImport.id;
		delete toImport.updated_time;
		delete toImport.created_time;
		delete toImport.encryption_applied;
		delete toImport.encryption_cipher_text;

		const folder = await this.restoreFolder();

		toImport.parent_id = folder.id;

		return Note.save(toImport);
	}

	public async maintenance() {
		this.maintenanceCalls_.push(true);
		try {
			const startTime = Date.now();
			logger.info('maintenance: Starting...');

			if (!Setting.value('revisionService.enabled')) {
				logger.info('maintenance: Service is disabled');
				// We do as if we had processed all the latest changes so that they can be cleaned up
				// later on by ItemChangeUtils.deleteProcessedChanges().
				Setting.setValue('revisionService.lastProcessedChangeId', await ItemChange.lastChangeId());
				await this.deleteOldRevisions(Setting.value('revisionService.ttlDays') * 24 * 60 * 60 * 1000);
			} else {
				logger.info('maintenance: Service is enabled');
				await this.collectRevisions();
				await this.deleteOldRevisions(Setting.value('revisionService.ttlDays') * 24 * 60 * 60 * 1000);

				logger.info(`maintenance: Done in ${Date.now() - startTime}ms`);
			}
		} catch (error) {
			logger.error('maintenance:', error);
		} finally {
			this.maintenanceCalls_.pop();
		}
	}

	public runInBackground(collectRevisionInterval: number = null) {
		if (this.isRunningInBackground_) return;
		this.isRunningInBackground_ = true;

		if (collectRevisionInterval === null) collectRevisionInterval = 1000 * 60 * 10;

		logger.info(`runInBackground: Starting background service with revision collection interval ${collectRevisionInterval}`);

		this.maintenanceTimer1_ = shim.setTimeout(() => {
			void this.maintenance();
		}, 1000 * 4);

		this.maintenanceTimer2_ = shim.setInterval(() => {
			void this.maintenance();
		}, collectRevisionInterval);
	}

	public async cancelTimers() {
		if (this.maintenanceTimer1_) {
			shim.clearTimeout(this.maintenanceTimer1_);
			this.maintenanceTimer1_ = null;
		}
		if (this.maintenanceTimer2_) {
			shim.clearInterval(this.maintenanceTimer2_);
			this.maintenanceTimer2_ = null;
		}

		return new Promise((resolve) => {
			const iid = shim.setInterval(() => {
				if (!this.maintenanceCalls_.length) {
					shim.clearInterval(iid);
					resolve(null);
				}
			}, 100);
		});
	}

	public async deleteHistoryForNote(noteIds: string | string[], options: DeleteOptions) {
		const ids = Array.isArray(noteIds) ? noteIds : [noteIds];
		await Revision.deleteHistoryForNote(ids, options);
		await this.resetHistoryState_(ids);
	}

	public async deleteUnencryptedHistoryForNote(noteIds: string | string[], options: DeleteOptions) {
		const ids = Array.isArray(noteIds) ? noteIds : [noteIds];
		await Revision.deleteUnencryptedHistoryForNote(ids, options);
		await this.resetHistoryState_(ids);
	}

	private async resetHistoryState_(noteIds: string[]) {
		// Clear any cached content in the item_changes table and reset the state of the note in the revision service, to ensure that any new revisions created
		// upon revision collection do not include contents which were present prior to triggering the deletion
		for (const noteId of noteIds) {
			await ItemChange.resetOldNoteContent(noteId);
			RevisionService.instance().removeChangedSinceCollection(noteId);
		}
	}
}