mynode/packages/lib/services/synchronizer/Synchronizer.conflicts.test.ts

import time from '../../time';
import { allNotesFolders, localNotesFoldersSameAsRemote } from '../../testing/test-utils-synchronizer';
import { synchronizerStart, setupDatabaseAndSynchronizer, sleep, switchClient, syncTargetId, loadEncryptionMasterKey, decryptionWorker, fileApi } from '../../testing/test-utils';
import Folder from '../../models/Folder';
import Note from '../../models/Note';
import BaseItem from '../../models/BaseItem';
import Setting from '../../models/Setting';
import { setEncryptionEnabled } from '../synchronizer/syncInfoUtils';
import { NoteEntity } from '../database/types';
import ItemChange from '../../models/ItemChange';

describe('Synchronizer.conflicts', () => {
	const createConflictNote = async (note: Partial<NoteEntity>) => {
		const originalNote = await Note.save(note);
		const conflictNote = await Note.createConflictNote(originalNote, ItemChange.SOURCE_SYNC);
		await Note.delete(originalNote.id, { trackDeleted: false });
		return conflictNote;
	};

	beforeEach(async () => {
		await setupDatabaseAndSynchronizer(1);
		await setupDatabaseAndSynchronizer(2);
		await switchClient(1);
	});

	it('should resolve note conflicts', (async () => {
		const folder1 = await Folder.save({ title: 'folder1' });
		const note1 = await Note.save({ title: 'un', parent_id: folder1.id });
		await synchronizerStart();

		await switchClient(2);

		await synchronizerStart();

		let syncItem = await BaseItem.syncItem(syncTargetId(), note1.id, { fields: ['sync_time'] });
		expect(syncItem.sync_time).toBe(note1.updated_time);

		let note2 = await Note.load(note1.id);
		note2.title = 'Updated on client 2';
		await Note.save(note2);
		note2 = await Note.load(note2.id);
		await synchronizerStart();

		await switchClient(1);

		let note2conf = await Note.load(note1.id);
		note2conf.title = 'Updated on client 1';
		await Note.save(note2conf);
		note2conf = await Note.load(note1.id);
		await synchronizerStart();

		syncItem = await BaseItem.syncItem(syncTargetId(), note2.id, { fields: ['sync_time'] });
		expect(syncItem.sync_time).toBe(note2.updated_time);

		const conflictedNotes = await Note.conflictedNotes();
		expect(conflictedNotes.length).toBe(1);

		// Other than the id (since the conflicted note is a duplicate), and the is_conflict property
		// the conflicted and original note must be the same in every way, to make sure no data has been lost.
		const conflictedNote = conflictedNotes[0];
		expect(conflictedNote.id === note2conf.id).toBe(false);
		expect(conflictedNote.conflict_original_id).toBe(note2conf.id);
		for (const n in conflictedNote) {
			if (!conflictedNote.hasOwnProperty(n)) continue;
			if (n === 'id' || n === 'is_conflict' || n === 'conflict_original_id') continue;
			expect((conflictedNote as Record<string, unknown>)[n]).toBe((note2conf as Record<string, unknown>)[n]);
		}

		const noteUpdatedFromRemote = await Note.load(note1.id);
		for (const n in noteUpdatedFromRemote) {
			if (!noteUpdatedFromRemote.hasOwnProperty(n)) continue;
			expect((noteUpdatedFromRemote as Record<string, unknown>)[n]).toBe((note2 as Record<string, unknown>)[n]);
		}
	}));

	it.each([
		['missing', false, ''],
		['empty', false, 'is_locked: \n'],
		['empty', true, 'is_locked: \n'],
	])('should discard the change of an older client that drops the lock of a note inside a share and push the local version back (%s is_locked, changed locally: %s)', (async (_name, changedLocally, lockLine) => {
		const folder = await Folder.save({ title: 'folder' });
		// Dated in the past, as is the older client's edit below, so the sync time cap does not hold back the push back.
		const note = await Note.save({ title: 'Locked', body: 'JLD01cipher', is_locked: 1, parent_id: folder.id, updated_time: time.unixMs() - 60_000 }, { autoTimestamp: false });
		await synchronizerStart();

		const path = `${note.id}.md`;
		const remote = (await fileApi().get(path)).replace('share_id: \n', 'share_id: share-1\n').replace('is_locked: 1\n', lockLine).replace('JLD01cipher', 'edited on an old client').replace(/^updated_time: .*$/m, `updated_time: ${time.unixMsToIso(note.updated_time + 1000)}`);
		await fileApi().put(path, remote);
		const title = changedLocally ? 'Renamed locally' : 'Locked';
		if (changedLocally) await Note.save({ id: note.id, title });
		await synchronizerStart();

		expect((await Note.conflictedNotes()).length).toBe(0);
		expect(await Note.load(note.id)).toMatchObject({ title, is_locked: 1, body: 'JLD01cipher' });

		await synchronizerStart();
		expect(await BaseItem.unserialize(await fileApi().get(path))).toMatchObject({ title, is_locked: 1, body: 'JLD01cipher' });
	}));

	it.each([false, true])('should keep a remote change that drops the lock of a note outside a share (changed locally: %s)', (async (changedLocally) => {
		const folder = await Folder.save({ title: 'folder' });
		const note = await Note.save({ title: 'Locked', body: 'JLD01cipher', is_locked: 1, parent_id: folder.id, updated_time: time.unixMs() - 60_000 }, { autoTimestamp: false });
		await synchronizerStart();

		const path = `${note.id}.md`;
		const remote = (await fileApi().get(path)).replace('is_locked: 1\n', '').replace('JLD01cipher', 'edited without the lock').replace(/^updated_time: .*$/m, `updated_time: ${time.unixMsToIso(note.updated_time + 1000)}`);
		await fileApi().put(path, remote);
		if (changedLocally) await Note.save({ id: note.id, title: 'Renamed locally' });
		await synchronizerStart();

		const bodies = [(await Note.load(note.id)).body, ...(await Note.conflictedNotes()).map(n => n.body)];
		expect(bodies).toContain('edited without the lock');
	}));

	it.each([
		['an older client edits an unlocked note', 0, ''],
		['another device removes the lock', 1, 'is_locked: 0\n'],
	])('should keep a remote change to a note inside a share when %s', (async (_name, isLocked, lockLine) => {
		const folder = await Folder.save({ title: 'folder' });
		const note = await Note.save({ title: 'note', body: 'JLD01cipher', is_locked: isLocked, parent_id: folder.id });
		await synchronizerStart();

		const path = `${note.id}.md`;
		const remote = (await fileApi().get(path)).replace('share_id: \n', 'share_id: share-1\n').replace(`is_locked: ${isLocked}\n`, lockLine).replace('JLD01cipher', 'changed remotely').replace(/^updated_time: .*$/m, `updated_time: ${time.unixMsToIso(note.updated_time + 1000)}`);
		await fileApi().put(path, remote);
		await synchronizerStart();

		expect(await Note.load(note.id)).toMatchObject({ is_locked: 0, body: 'changed remotely' });
	}));

	it('should not keep a conflict when a shared locked note arrives with its lock state', (async () => {
		const folder = await Folder.save({ title: 'folder' });
		const note = await Note.save({ title: 'Locked', body: 'JLD01cipher', is_locked: 1, parent_id: folder.id });
		await synchronizerStart();

		const path = `${note.id}.md`;
		const remote = (await fileApi().get(path)).replace('share_id: \n', 'share_id: share-1\n').replace('JLD01cipher', 'JLD01ciphertext2').replace(/^updated_time: .*$/m, `updated_time: ${time.unixMsToIso(note.updated_time + 1000)}`);
		await fileApi().put(path, remote);
		await synchronizerStart();

		expect((await Note.conflictedNotes()).length).toBe(0);
		expect(await Note.load(note.id)).toMatchObject({ is_locked: 1, body: 'JLD01ciphertext2' });
	}));

	it('should cap sync_time to the current device time in the delta step', (async () => {
		const folder1 = await Folder.save({ title: 'folder1' });
		const note1 = await Note.save({ title: 'un', parent_id: folder1.id, updated_time: time.unixMs() + 100_000 }, { autoTimestamp: false });
		await synchronizerStart();

		await switchClient(2);

		await synchronizerStart();

		const syncItem = await BaseItem.syncItem(syncTargetId(), note1.id, { fields: ['sync_time'] });
		expect(syncItem.sync_time).toBeLessThan(note1.updated_time);
	}));

	it('should leave changes made after the upload phase for the next sync', (async () => {
		const folder = await Folder.save({ title: 'folder' });
		const note = await Note.save({ title: 'original', parent_id: folder.id });
		await synchronizerStart();

		await switchClient(2);
		await synchronizerStart();
		await sleep(0.1);
		await Note.save({ id: note.id, title: 'remote change' });
		await synchronizerStart();

		await switchClient(1);
		let localChange: NoteEntity = null;
		const loadItemsByIds = BaseItem.loadItemsByIds.bind(BaseItem);
		const loadItemsByIdsMock = jest.spyOn(BaseItem, 'loadItemsByIds').mockImplementation(async ids => {
			const items = await loadItemsByIds(ids);
			if (ids.includes(note.id)) {
				await sleep(0.1);
				localChange = await Note.save({ id: note.id, title: 'local change' });
			}
			return items;
		});

		await synchronizerStart(null, { syncSteps: ['delta'] });
		loadItemsByIdsMock.mockRestore();

		expect((await Note.load(note.id)).title).toBe('local change');
		const syncItem = await BaseItem.syncItem(syncTargetId(), note.id, { fields: ['sync_time'] });
		expect(syncItem.sync_time).toBeLessThan(localChange.updated_time);

		// The following upload phase sees both changes and uses the existing
		// conflict handling path.
		await synchronizerStart();
		expect((await Note.load(note.id)).title).toBe('remote change');
		expect((await Note.conflictedNotes()).map(note => note.title)).toContain('local change');
	}));

	it('should resolve folders conflicts', (async () => {
		const folder1 = await Folder.save({ title: 'folder1' });
		await Note.save({ title: 'un', parent_id: folder1.id });
		await synchronizerStart();

		await switchClient(2); // ----------------------------------

		await synchronizerStart();

		await sleep(0.1);

		let folder1_modRemote = await Folder.load(folder1.id);
		folder1_modRemote.title = 'folder1 UPDATE CLIENT 2';
		await Folder.save(folder1_modRemote);
		folder1_modRemote = await Folder.load(folder1_modRemote.id);

		await synchronizerStart();

		await switchClient(1); // ----------------------------------

		await sleep(0.1);

		let folder1_modLocal = await Folder.load(folder1.id);
		folder1_modLocal.title = 'folder1 UPDATE CLIENT 1';
		await Folder.save(folder1_modLocal);
		folder1_modLocal = await Folder.load(folder1.id);

		await synchronizerStart();

		const folder1_final = await Folder.load(folder1.id);
		expect(folder1_final.title).toBe(folder1_modRemote.title);
	}));

	it('should resolve conflict if remote folder has been deleted, but note has been added to folder locally', (async () => {
		const folder1 = await Folder.save({ title: 'folder1' });
		await synchronizerStart();

		await switchClient(2);

		await synchronizerStart();
		await Folder.delete(folder1.id);
		await synchronizerStart();

		await switchClient(1);

		await Note.save({ title: 'note1', parent_id: folder1.id });
		await synchronizerStart();
		const items: NoteEntity[] = await allNotesFolders();
		expect(items.length).toBe(1);
		expect(items[0].title).toBe('note1');
		expect(items[0].is_conflict).toBe(1);
	}));

	it('should resolve conflict if note has been deleted remotely and locally', (async () => {
		const folder = await Folder.save({ title: 'folder' });
		const note = await Note.save({ title: 'note', parent_id: folder.title });
		await synchronizerStart();

		await switchClient(2);

		await synchronizerStart();
		await Note.delete(note.id);
		await synchronizerStart();

		await switchClient(1);

		await Note.delete(note.id);
		await synchronizerStart();

		const items = await allNotesFolders();
		expect(items.length).toBe(1);
		expect(items[0].title).toBe('folder');

		await localNotesFoldersSameAsRemote(items, expect);
	}));

	it('should handle conflict when remote note is deleted then local note is modified', (async () => {
		const folder1 = await Folder.save({ title: 'folder1' });
		const note1 = await Note.save({ title: 'un', parent_id: folder1.id });
		await synchronizerStart();

		await switchClient(2);

		await synchronizerStart();

		await sleep(0.1);

		await Note.delete(note1.id);

		await synchronizerStart();

		await switchClient(1);

		const newTitle = 'Modified after having been deleted';
		await Note.save({ id: note1.id, title: newTitle });

		await synchronizerStart();

		const conflictedNotes = await Note.conflictedNotes();

		expect(conflictedNotes.length).toBe(1);
		expect(conflictedNotes[0].title).toBe(newTitle);

		const unconflictedNotes = await Note.unconflictedNotes();

		expect(unconflictedNotes.length).toBe(0);
	}));

	it('should handle conflict when remote folder is deleted then local folder is renamed', (async () => {
		const folder1 = await Folder.save({ title: 'folder1' });
		const folder2 = await Folder.save({ title: 'folder2', parent_id: folder1.id });
		const note1 = await Note.save({ title: 'note1', parent_id: folder2.id });
		const note2 = await Note.save({ title: 'note2', parent_id: folder1.id });
		await synchronizerStart();

		await switchClient(2);

		await synchronizerStart();

		await sleep(0.1);

		await Folder.delete(folder1.id, { deleteChildren: false });

		await synchronizerStart();

		await switchClient(1);

		await sleep(0.1);

		const newTitle = 'Modified after having been deleted';
		await Folder.save({ id: folder1.id, title: newTitle });

		await synchronizerStart();

		const remainingFolder1 = await Folder.load(folder1.id);
		const remainingFolder2 = await Folder.load(folder2.id);
		const remainingNote1 = await Note.load(note1.id);
		const remainingNote2 = await Note.load(note2.id);

		expect(remainingFolder1).toBeUndefined();
		expect(remainingFolder2?.id).toBe(folder2.id);
		expect(remainingNote1?.id).toBe(note1.id);
		expect(remainingNote2?.id).toBe(note2.id);
	}));

	it('should sync newly created conflict notes', (async () => {
		const f1 = await Folder.save({ title: 'folder' });
		await createConflictNote({ title: 'mynote', parent_id: f1.id });
		await synchronizerStart();

		await switchClient(2);

		await synchronizerStart();
		const notes = await Note.all();
		const folders = await Folder.all();
		expect(notes).toHaveLength(1);
		expect(notes[0].is_conflict).toBe(1);
		expect(folders.length).toBe(1);
	}));

	it('should retain the local version when a synced conflict note has conflicting changes', async () => {
		const folder = await Folder.save({ title: 'folder' });
		const conflictNote = await createConflictNote({ title: 'original', parent_id: folder.id });
		await synchronizerStart();

		await switchClient(2);
		await synchronizerStart();
		await sleep(0.1);
		await Note.save({ id: conflictNote.id, title: 'remote change' });
		await synchronizerStart();

		await switchClient(1);
		await sleep(0.1);
		await Note.save({ id: conflictNote.id, title: 'local change' });
		await synchronizerStart();
		expect((await Note.load(conflictNote.id)).title).toBe('local change');

		await switchClient(2);
		await synchronizerStart();
		expect((await Note.load(conflictNote.id)).title).toBe('local change');
	});

	it('should use the remote version when it is newer than a local synced conflict note change', async () => {
		const folder = await Folder.save({ title: 'folder' });
		const conflictNote = await createConflictNote({ title: 'original', parent_id: folder.id });
		await synchronizerStart();

		await switchClient(2);
		await synchronizerStart();

		await switchClient(1);
		await sleep(0.1);
		await Note.save({ id: conflictNote.id, title: 'local change' });

		await switchClient(2);
		await sleep(0.1);
		await Note.save({ id: conflictNote.id, title: 'remote change' });
		await synchronizerStart();

		await switchClient(1);
		await synchronizerStart();
		expect((await Note.load(conflictNote.id)).title).toBe('remote change');

		await switchClient(2);
		await synchronizerStart();
		expect((await Note.load(conflictNote.id)).title).toBe('remote change');
	});

	it('should let remote deletion win over a local conflict note update', async () => {
		const folder = await Folder.save({ title: 'folder' });
		const conflictNote = await createConflictNote({ title: 'original', parent_id: folder.id });
		await synchronizerStart();

		await switchClient(2);
		await synchronizerStart();
		await Note.delete(conflictNote.id);
		await synchronizerStart();

		await switchClient(1);
		await sleep(0.1);
		await Note.save({ id: conflictNote.id, title: 'local change' });
		await synchronizerStart();

		expect(await Note.load(conflictNote.id)).toBeUndefined();
		expect(await Note.conflictedNotes()).toHaveLength(0);
	});

	it('should delete remotely synced conflict notes', (async () => {
		const f1 = await Folder.save({ title: 'folder' });
		const n1 = await createConflictNote({ title: 'mynote', parent_id: f1.id });
		await synchronizerStart();

		await switchClient(2);

		await synchronizerStart();
		await Note.save({ id: n1.id, is_conflict: 1 });
		await Note.delete(n1.id);
		const deletedItems = await BaseItem.deletedItems(syncTargetId());

		expect(deletedItems).toHaveLength(1);
		expect(deletedItems[0].item_id).toBe(n1.id);

		await synchronizerStart();
		await switchClient(1);
		await synchronizerStart();
		expect(await Note.load(n1.id)).toBeUndefined();
	}));

	async function ignorableNoteConflictTest(withEncryption: boolean) {
		// This covers the conflict behaviour without auto-merge, which would otherwise
		// merge the encrypted case instead of creating a conflict note
		Setting.setValue('sync.autoMergeConflicts', false);

		if (withEncryption) {
			setEncryptionEnabled(true);
			await loadEncryptionMasterKey();
		}

		const folder1 = await Folder.save({ title: 'folder1' });
		const note1 = await Note.save({ title: 'un', is_todo: 1, parent_id: folder1.id });
		await synchronizerStart();

		await switchClient(2);

		await synchronizerStart();
		if (withEncryption) {
			await loadEncryptionMasterKey(null, true);
			await decryptionWorker().start();
		}
		let note2 = await Note.load(note1.id);
		note2.todo_completed = time.unixMs() - 1;
		await Note.save(note2);
		note2 = await Note.load(note2.id);
		await synchronizerStart();

		await switchClient(1);

		let note2conf = await Note.load(note1.id);
		note2conf.todo_completed = time.unixMs();
		await Note.save(note2conf);
		note2conf = await Note.load(note1.id);
		await synchronizerStart();
		if (withEncryption) await decryptionWorker().start();

		// This is a metadata-only conflict: Both clients marked the same todo as done,
		// but at slightly different times. Once encrypted remote content is decrypted,
		// it can be ignored in the same way as an unencrypted conflict.
		const conflictedNotes = await Note.conflictedNotes();
		expect(conflictedNotes.length).toBe(0);

		const notes = await Note.all();
		expect(notes.length).toBe(1);
		expect(notes[0].id).toBe(note1.id);
		expect(notes[0].todo_completed).toBe(note2.todo_completed);
	}

	it('should not consider it is a conflict if neither the title nor body of the note have changed', (async () => {
		await ignorableNoteConflictTest(false);
	}));

	it('should ignore metadata-only conflicts when the remote note is encrypted', (async () => {
		await ignorableNoteConflictTest(true);
	}));

});