feat: Initial clean release of CHM

packages/lib/services/ocr/drivers/OcrDriverTesseract.ts

@@ -0,0 +1,263 @@
+import { RecognizeResult, RecognizeResultBoundingBox, RecognizeResultLine, RecognizeResultWord } from '../utils/types';
+import { Worker, WorkerOptions, createWorker, RecognizeResult as TesseractRecognizeResult, OEM } from 'tesseract.js';
+import OcrDriverBase from '../OcrDriverBase';
+import { Minute } from '@joplin/utils/time';
+import shim from '../../../shim';
+import Logger from '@joplin/utils/Logger';
+import filterOcrText from '../utils/filterOcrText';
+import Resource from '../../../models/Resource';
+import { ResourceOcrDriverId, ResourceOcrStatus } from '../../database/types';
+
+const logger = Logger.create('OcrDriverTesseract');
+
+interface Tesseract {
+	createWorker: typeof createWorker;
+}
+
+interface WorkerWrapper {
+	id: number;
+	busy: boolean;
+	instance: Worker;
+}
+
+let workerId_ = 1;
+
+const formatTesseractBoundingBox = (boundingBox: Tesseract.Bbox): RecognizeResultBoundingBox => {
+	return [boundingBox.x0, boundingBox.x1, boundingBox.y0, boundingBox.y1];
+};
+
+// 2023-12-13: Empirically, it seems anything below 70 is not usable. Between 70
+// and 75 it's hit and miss, but often it's good enough that we should keep the result.
+// Above this is usually reliable. Using 70 for now.
+//
+// 2025-04-03: Changed to 55 to detect text in images that are supported in
+// other tools but were not in Joplin.
+//
+// https://github.com/laurent22/joplin/issues/11608
+const minConfidence = 55;
+
+interface Options {
+	workerPath: string;
+	corePath: string;
+	languageDataPath: string|null;
+}
+
+export default class OcrDriverTesseract extends OcrDriverBase {
+
+	private tesseract_: Tesseract = null;
+	private workerPath_: string;
+	private corePath_: string;
+	private languageDataPath_: string|null = null;
+	private workers_: Record<string, WorkerWrapper[]> = {};
+
+	public constructor(tesseract: Tesseract, { workerPath, corePath, languageDataPath }: Options) {
+		super();
+		this.tesseract_ = tesseract;
+		this.workerPath_ = workerPath;
+		this.corePath_ = corePath;
+		this.languageDataPath_ = languageDataPath;
+	}
+
+	public get driverId() {
+		return ResourceOcrDriverId.PrintedText;
+	}
+
+	public static async clearLanguageDataCache() {
+		if (typeof indexedDB === 'undefined') {
+			throw new Error('Missing indexedDB access!');
+		}
+
+		logger.info('Clearing cached language data...');
+
+		const requestAsPromise = <T> (request: IDBRequest) => {
+			return new Promise<T>((resolve, reject) => {
+				request.addEventListener('success', () => { resolve(request.result); });
+				request.addEventListener('error', (event) => {
+					if ('error' in event) {
+						reject(new Error(`Request failed: ${event.error}`));
+					} else {
+						reject(new Error('Request failed with unknown error.'));
+					}
+				});
+			});
+		};
+
+		const db = await requestAsPromise<IDBDatabase>(indexedDB.open('keyval-store'));
+		const getStore = (mode: IDBTransactionMode) => {
+			return db.transaction(['keyval'], mode).objectStore('keyval');
+		};
+
+		const allKeys = await requestAsPromise<string[]>(getStore('readonly').getAllKeys());
+		const languageDataExtension = '.traineddata';
+		const keysToClear = allKeys.filter(key => key.endsWith(languageDataExtension));
+		for (const key of keysToClear) {
+			logger.info('Clearing language data with key', key);
+			await requestAsPromise(getStore('readwrite').delete(key));
+		}
+	}
+
+	private async acquireWorker(language: string) {
+		if (!this.workers_[language]) this.workers_[language] = [];
+
+		const existingWorker = this.workers_[language].find(w => !w.busy);
+
+		if (existingWorker) {
+			existingWorker.busy = true;
+			return existingWorker;
+		}
+
+		const createWorkerOptions: Partial<WorkerOptions> = {
+			workerBlobURL: false,
+
+			// Sometimes Tesseract stops working (especially in dev mode?) as the worker is stuck
+			// loading the language file. In that case, setting the cacheMode to "none" fixes the
+			// issue.
+
+			// cacheMethod: 'none',
+		};
+
+		if (this.workerPath_) createWorkerOptions.workerPath = this.workerPath_;
+		if (this.corePath_) createWorkerOptions.corePath = this.corePath_;
+		if (this.languageDataPath_) createWorkerOptions.langPath = this.languageDataPath_;
+
+		const worker = await this.tesseract_.createWorker(language, OEM.LSTM_ONLY, createWorkerOptions);
+
+		const output: WorkerWrapper = {
+			id: workerId_++,
+			instance: worker,
+			busy: true,
+		};
+
+		logger.info(`Created worker: ${output.id}`);
+
+		this.workers_[language].push(output);
+
+		return output;
+	}
+
+	public async dispose() {
+		for (const [language, workers] of Object.entries(this.workers_)) {
+			for (const w of workers) {
+				await w.instance.terminate();
+			}
+			this.workers_[language] = [];
+		}
+	}
+
+	private async terminateWorker(id: number) {
+		for (const [, workers] of Object.entries(this.workers_)) {
+			const idx = workers.findIndex(w => w.id === id);
+			if (idx < 0) continue;
+
+			await workers[idx].instance.terminate();
+			workers.splice(idx, 1);
+			break;
+		}
+	}
+
+	private async releaseWorker(worker: WorkerWrapper) {
+		worker.busy = false;
+	}
+
+	public async recognize(language: string, filePath: string): Promise<RecognizeResult> {
+		// eslint-disable-next-line no-async-promise-executor -- can't think of any way to handle the timeout without using `new Promise`
+		return new Promise(async (resolve, reject) => {
+			const worker = await this.acquireWorker(language);
+
+			let hasTimedOut = false;
+			const terminateTimeout_ = shim.setTimeout(async () => {
+				await this.terminateWorker(worker.id);
+				hasTimedOut = true;
+				reject(new Error(`Recognize operation timed out on: ${filePath}`));
+			}, 10 * Minute);
+
+			let result: TesseractRecognizeResult = null;
+
+			try {
+				result = await worker.instance.recognize(filePath, {}, { text: false, blocks: true });
+			} catch (e) {
+				const error: Error = typeof e === 'string' ? new Error(e) : e;
+				error.message = `Recognition failed on: ${filePath}: ${error.message}`;
+				if (!hasTimedOut) reject(error);
+				return;
+			}
+
+			if (hasTimedOut) return;
+
+			shim.clearTimeout(terminateTimeout_);
+
+			await this.releaseWorker(worker);
+
+			interface GoodParagraph {
+				text: string;
+			}
+
+			const goodParagraphs: GoodParagraph[] = [];
+			let goodLines: RecognizeResultLine[] = [];
+
+			for (const block of result.data.blocks) {
+
+				for (const paragraph of block.paragraphs) {
+					const lines: RecognizeResultLine[] = [];
+
+					for (const line of paragraph.lines) {
+						// If the line confidence is above the threshold we keep the
+						// whole text. The confidence of individual words will vary and
+						// may be below the treshold, but there's a chance they will
+						// still be correct if the line as a whole is well recognised.
+						if (line.confidence < minConfidence) continue;
+
+						const lineBaselineAt = (x: number, top: boolean) => {
+							const dy = line.baseline.y1 - line.baseline.y0;
+							const dx = line.baseline.x1 - line.baseline.x0;
+							// Avoid division by zero
+							if (dx === 0) {
+								return top ? line.baseline.y0 : line.baseline.y1;
+							} else {
+								const slope = dy / dx;
+								return slope * (x - line.baseline.x0) + line.baseline.y0;
+							}
+						};
+
+						const goodWords: RecognizeResultWord[] = line.words
+							.map(w => {
+								const baselineX1 = w.bbox.x0;
+								const baselineY1 = lineBaselineAt(baselineX1, true);
+								const baselineX2 = w.bbox.x1;
+								const baselineY2 = lineBaselineAt(baselineX2, false);
+
+								const output: RecognizeResultWord = {
+									t: w.text,
+									bb: formatTesseractBoundingBox(w.bbox),
+									bl: [baselineX1, baselineX2, baselineY1, baselineY2],
+								};
+
+								return output;
+							});
+
+						lines.push({
+							words: goodWords,
+						});
+					}
+
+					goodParagraphs.push({
+						text: lines.map(l => l.words.map(w => w.t).join(' ')).join('\n'),
+					});
+
+					goodLines = goodLines.concat(lines);
+				}
+			}
+
+			resolve({
+				// Note that Tesseract provides a `.text` property too, but it's the
+				// concatenation of all lines, even those with a low confidence
+				// score, so we recreate it here based on the good lines.
+				ocr_text: filterOcrText(goodParagraphs.map(p => p.text).join('\n')),
+				ocr_details: Resource.serializeOcrDetails(goodLines),
+				ocr_status: ResourceOcrStatus.Done,
+				ocr_error: '',
+			});
+		});
+	}
+
+}

packages/lib/services/ocr/drivers/OcrDriverTranscribe.test.ts

@@ -0,0 +1,107 @@
+import Setting from '../../../models/Setting';
+import { createNoteAndResource, setupDatabaseAndSynchronizer, switchClient } from '../../../testing/test-utils';
+import { ResourceOcrStatus } from '../../database/types';
+import OcrDriverTranscribe from './OcrDriverTranscribe';
+import { reg } from '../../../registry';
+
+type JobGenerated = { jobId: string };
+type GetResultPending = { state: string; jobId: string };
+type GetResultCompleted = { state: 'completed'; jobId: string; output: { result: string } };
+type GetResultFailed = { state: 'failed'; jobId: string; output: { stack: string; message: string } };
+
+type Response = JobGenerated | GetResultPending | GetResultCompleted | GetResultFailed | Error;
+
+interface MockApi {
+	exec: jest.MockedFunction<(
+		method: string,
+		path: string,
+		query?: unknown,
+		body?: unknown,
+		headers?: Record<string, string>,
+		options?: Record<string, unknown>,
+	)=> Promise<Response>>;
+}
+
+describe('OcrDriverTranscribe', () => {
+	let mockApi: MockApi;
+
+	beforeEach(async () => {
+		await setupDatabaseAndSynchronizer(1);
+		await switchClient(1);
+
+		mockApi = {
+			exec: jest.fn(),
+		};
+
+		const mockApiMethod = jest.fn().mockResolvedValue(mockApi);
+		const mockDriver = { api: mockApiMethod };
+		const mockFileApi = { driver: jest.fn().mockReturnValue(mockDriver) };
+		const mockSyncTarget = { fileApi: jest.fn().mockResolvedValue(mockFileApi) };
+
+		reg.syncTarget = jest.fn().mockReturnValue(mockSyncTarget);
+	});
+
+	it('should return an error if synchronization target is not set', async () => {
+		const { resource } = await createNoteAndResource();
+		const htr = new OcrDriverTranscribe();
+		const response = await htr.recognize('', 'mock-path', resource.id);
+
+		expect(response.ocr_status).toBe(ResourceOcrStatus.Error);
+	});
+
+	it('should return correct response when successful', async () => {
+		const { resource } = await createNoteAndResource();
+
+		mockApi.exec.mockResolvedValue(Promise.resolve({ jobId: 'not-a-real-job-id' }));
+		mockApi.exec.mockResolvedValue(Promise.resolve({ state: 'pending', jobId: 'not-a-real-job-id' }));
+		mockApi.exec.mockResolvedValue(Promise.resolve({ state: 'completed', jobId: 'not-a-real-job-id', output: { result: 'this is the final transcription' } }));
+
+		const htr = new OcrDriverTranscribe([1]);
+		Setting.setValue('sync.target', 9);
+
+		const response = await htr.recognize('', resource.filename, resource.id);
+
+		expect(response.ocr_status).toBe(ResourceOcrStatus.Done);
+		expect(response.ocr_text).toBe('this is the final transcription');
+	});
+
+	it('should return error when unsuccessful', async () => {
+		const { resource } = await createNoteAndResource();
+
+		mockApi.exec.mockResolvedValue(Promise.resolve({ jobId: 'not-a-real-job-id' }));
+		mockApi.exec.mockResolvedValue(Promise.resolve({ state: 'failed', jobId: 'not-a-real-job-id', output: { stack: '', message: 'Something went wrong' } }));
+
+		const htr = new OcrDriverTranscribe([1]);
+		Setting.setValue('sync.target', 9);
+
+		const response = await htr.recognize('', resource.filename, resource.id);
+
+		expect(response.ocr_status).toBe(ResourceOcrStatus.Error);
+		expect(response.ocr_error).toEqual({ stack: '', message: 'Something went wrong' });
+	});
+
+	it('should be able to retrieve jobId from database instead of creating a new job', async () => {
+		const { resource } = await createNoteAndResource();
+		const jobId = 'jobIdThat should be reused latter';
+
+		mockApi.exec.mockResolvedValue(Promise.resolve({ jobId }));
+		mockApi.exec.mockImplementationOnce(() => { throw new Error('Network request failed'); });
+
+		const htr = new OcrDriverTranscribe([1]);
+		Setting.setValue('sync.target', 9);
+
+		const response = await htr.recognize('', resource.filename, resource.id);
+		await htr.dispose();
+		expect(response.ocr_status).toBe(ResourceOcrStatus.Todo);
+		expect(response.ocr_error).toBe('');
+
+		// Simulating closing/opening application
+		mockApi.exec.mockResolvedValue({ jobId, state: 'completed', output: { result: 'result' } });
+		const htr2 = new OcrDriverTranscribe([1]);
+
+		const response2 = await htr2.recognize('', resource.filename, resource.id);
+		expect(response2.ocr_status).toBe(ResourceOcrStatus.Done);
+		expect(response2.ocr_text).toBe('result');
+
+	});
+});

packages/lib/services/ocr/drivers/OcrDriverTranscribe.ts

@@ -0,0 +1,135 @@
+import { emptyRecognizeResult, RecognizeResult } from '../utils/types';
+import OcrDriverBase from '../OcrDriverBase';
+import Logger from '@joplin/utils/Logger';
+import { ResourceOcrDriverId, ResourceOcrStatus } from '../../database/types';
+import KvStore from '../../KvStore';
+import shim from '../../../shim';
+import { msleep } from '@joplin/utils/time';
+import Resource from '../../../models/Resource';
+import { reg } from '../../../registry';
+
+const logger = Logger.create('OcrDriverTranscribe');
+
+type CreateJobResult = { jobId: string };
+
+export default class OcrDriverTranscribe extends OcrDriverBase {
+
+	private retryIntervals_ = [10 * 1000, 15 * 1000, 30 * 1000, 60 * 1000];
+	private jobIdKeyPrefix_ = 'OcrDriverTranscribe::JobId::';
+	private disposed_ = false;
+
+	public constructor(interval?: number[]) {
+		super();
+		this.retryIntervals_ = interval ?? this.retryIntervals_;
+	}
+
+	public get driverId() {
+		return ResourceOcrDriverId.HandwrittenText;
+	}
+
+	public async recognize(_language: string, filePath: string, resourceId: string): Promise<RecognizeResult> {
+		logger.info(`${resourceId}: Starting to recognize resource from ${filePath}`);
+
+		const key = `${this.jobIdKeyPrefix_}${resourceId}`;
+		const storedJobId = await KvStore.instance().value(key);
+		let jobId = typeof storedJobId === 'string' ? storedJobId : null;
+
+		try {
+			if (!jobId) {
+				await Resource.save({
+					id: resourceId,
+					ocr_status: ResourceOcrStatus.Processing,
+				});
+				logger.info(`${resourceId}: Job does not exist yet, creating...`);
+				jobId = await this.queueJob(filePath, resourceId);
+
+				logger.info(`${resourceId}: Job created, reference: ${jobId}`);
+				await KvStore.instance().setValue(key, jobId);
+			}
+
+			const ocrResult = await this.checkJobIsFinished(jobId, resourceId);
+			await KvStore.instance().deleteValue(key);
+
+			return {
+				...emptyRecognizeResult(),
+				...ocrResult,
+			};
+		} catch (error) {
+			if (shim.fetchRequestCanBeRetried(error) || error.code === 503) {
+				return emptyRecognizeResult();
+			}
+			await KvStore.instance().deleteValue(key);
+			return {
+				...emptyRecognizeResult(),
+				ocr_status: ResourceOcrStatus.Error,
+				ocr_error: error.message,
+			};
+		}
+	}
+
+	private async queueJob(filePath: string, resourceId: string) {
+		const api = await this.api();
+
+		const result: CreateJobResult = await api.exec('POST', 'api/transcribe', null, null, {
+			'Content-Type': 'application/octet-stream',
+		}, { path: filePath, source: 'file' });
+
+		logger.info(`${resourceId}: Job queued`);
+		return result.jobId;
+	}
+
+	private async checkJobIsFinished(jobId: string, resourceId: string) {
+		logger.info(`${resourceId}: Checking if job is finished...`);
+		let i = 0;
+		while (true) {
+			if (this.disposed_) break;
+
+			const api = await this.api();
+
+			const response = await api.exec('GET', `api/transcribe/${jobId}`);
+
+			if (this.disposed_) break;
+
+			if (response.state === 'completed') {
+				logger.info(`${resourceId}: Finished.`);
+				return {
+					ocr_status: ResourceOcrStatus.Done,
+					ocr_text: response.output.result,
+				};
+			} else if (response.state === 'failed') {
+				logger.info(`${resourceId}: Failed.`);
+				return {
+					ocr_status: ResourceOcrStatus.Error,
+					ocr_error: response.output,
+				};
+			}
+
+			logger.info(`${resourceId}: Job not finished yet, waiting... ${this.getInterval(i)}`);
+			await msleep(this.getInterval(i));
+			i += 1;
+		}
+
+		return {
+			ocr_status: ResourceOcrStatus.Error,
+			ocr_error: 'OcrDriverTranscribe was stopped while waiting for a transcription',
+		};
+	}
+
+	private getInterval(index: number) {
+		if (index >= this.retryIntervals_.length) {
+			return this.retryIntervals_[this.retryIntervals_.length - 1];
+		}
+		return this.retryIntervals_[index];
+	}
+
+	private async api() {
+		const fileApi = await reg.syncTarget().fileApi();
+		return fileApi.driver().api();
+	}
+
+	public dispose() {
+		this.disposed_ = true;
+		return Promise.resolve();
+	}
+
+}