feat: Initial clean release of CHM

packages/app-mobile/components/voiceTyping/AudioRecordingBanner.tsx

@@ -0,0 +1,255 @@
+import * as React from 'react';
+import { PrimaryButton, SecondaryButton } from '../buttons';
+import { _ } from '@joplin/lib/locale';
+import { useCallback, useEffect, useRef, useState } from 'react';
+import { AudioQuality, getRecordingPermissionsAsync, IOSOutputFormat, requestRecordingPermissionsAsync, setAudioModeAsync, type RecordingOptions, useAudioRecorder as useExpoAudioRecorder, useAudioRecorderState, requestNotificationPermissionsAsync } from 'expo-audio';
+import Logger from '@joplin/utils/Logger';
+import { OnFileSavedCallback, RecorderState } from './types';
+import { Platform } from 'react-native';
+import shim from '@joplin/lib/shim';
+import FsDriverWeb from '../../utils/fs-driver/fs-driver-rn.web';
+import uuid from '@joplin/lib/uuid';
+import RecordingControls from './RecordingControls';
+import { Text } from 'react-native-paper';
+import time from '@joplin/lib/time';
+import { toFileExtension } from '@joplin/lib/mime-utils';
+import { formatMsToDurationCompat, msleep } from '@joplin/utils/time';
+
+const logger = Logger.create('AudioRecording');
+
+interface Props {
+	onFileSaved: OnFileSavedCallback;
+	onDismiss: ()=> void;
+}
+
+// Modified from the Expo default recording options to create
+// .m4a recordings on both Android and iOS (rather than .3gp on Android).
+const recordingOptions: RecordingOptions = {
+	extension: '.m4a',
+	isMeteringEnabled: true,
+	sampleRate: 44100,
+	numberOfChannels: 2,
+	bitRate: 64000,
+	android: {
+		extension: '.m4a',
+		outputFormat: 'mpeg4',
+		audioEncoder: 'aac',
+	},
+	ios: {
+		extension: '.m4a',
+		audioQuality: AudioQuality.MIN,
+		outputFormat: IOSOutputFormat.MPEG4AAC,
+		linearPCMBitDepth: 16,
+		linearPCMIsBigEndian: false,
+		linearPCMIsFloat: false,
+	},
+	web: Platform.OS === 'web' ? {
+		mimeType: [
+			// Different browsers support different audio formats.
+			// In most cases, prefer audio/ogg and audio/mp4 to audio/webm because
+			// Chrome and Firefox create .webm files without duration information.
+			// See https://issues.chromium.org/issues/40482588
+			'audio/ogg', 'audio/mp4', 'audio/webm',
+		].find(type => MediaRecorder.isTypeSupported(type)) ?? 'audio/webm',
+		bitsPerSecond: 128000,
+	} : {},
+};
+
+const getRecordingFileName = (extension: string) => {
+	return `recording-${time.formatDateToLocal(new Date())}${extension}`;
+};
+
+const recordingToSaveData = async (recordingUri: string|null) => {
+	if (!recordingUri) throw new Error(_('Unable to access the recording file.'));
+
+	let uri = recordingUri;
+	let type: string|undefined;
+	let fileName;
+
+	if (Platform.OS === 'web') {
+		// On web, we need to fetch the result (which is a blob URL) and save it in our
+		// virtual file system so that it can be processed elsewhere.
+		const fetchResult = await fetch(uri);
+		const blob = await fetchResult.blob();
+
+		type = recordingOptions.web.mimeType;
+		const extension = `.${toFileExtension(type)}`;
+		fileName = getRecordingFileName(extension);
+		const file = new File([blob], fileName);
+
+		const path = `/tmp/${uuid.create()}-${fileName}`;
+		await (shim.fsDriver() as FsDriverWeb).createReadOnlyVirtualFile(path, file);
+		uri = path;
+	} else {
+		const extension = Platform.select({
+			android: recordingOptions.android.extension,
+			ios: recordingOptions.ios.extension,
+			default: '',
+		});
+		fileName = getRecordingFileName(extension);
+	}
+
+	return { uri, fileName, type };
+};
+
+const resetAudioMode = async () => {
+	await setAudioModeAsync({
+		allowsRecording: false,
+		allowsBackgroundRecording: false,
+		playsInSilentMode: false,
+		shouldPlayInBackground: false,
+	});
+};
+
+const useAudioRecorder = (onFileSaved: OnFileSavedCallback, onDismiss: ()=> void) => {
+	const [recordingState, setRecordingState] = useState<RecorderState>(RecorderState.Idle);
+	const [error, setError] = useState('');
+	const recorder = useExpoAudioRecorder(recordingOptions);
+	const recorderStatus = useAudioRecorderState(recorder, 100);
+	const isRecordingRef = useRef(false);
+
+	const onStartRecording = useCallback(async () => {
+		try {
+			setRecordingState(RecorderState.Loading);
+			setError('');
+
+			const permissionResponse = await getRecordingPermissionsAsync();
+			if (permissionResponse.status !== 'granted') {
+				const response = await requestRecordingPermissionsAsync();
+				if (!response.granted) {
+					throw new Error(_('Missing permission to record audio.'));
+				}
+
+				// Work around "This experience is currently in the background, so the audio session could not be activated"
+				// See https://github.com/expo/expo/issues/21782
+				await msleep(500);
+			}
+
+			let allowsBackgroundRecording = true;
+			if (Platform.OS === 'android') {
+				const notificationPermission = await requestNotificationPermissionsAsync();
+				if (!notificationPermission.granted) {
+					allowsBackgroundRecording = false;
+				}
+			}
+
+			await setAudioModeAsync({
+				allowsRecording: true,
+				allowsBackgroundRecording,
+				playsInSilentMode: true,
+				shouldPlayInBackground: true,
+				// Fixes an issue where opening a recording in the iOS audio player
+				// breaks creating new recordings.
+				// See https://github.com/expo/expo/issues/31152#issuecomment-2341811087
+				interruptionMode: 'doNotMix',
+			});
+			await recorder.prepareToRecordAsync();
+			isRecordingRef.current = true;
+			recorder.record();
+			setRecordingState(RecorderState.Recording);
+		} catch (error) {
+			logger.error('Error starting recording:', error);
+			setError(`Recording error: ${error}`);
+			setRecordingState(RecorderState.Error);
+
+			if (isRecordingRef.current) {
+				isRecordingRef.current = false;
+				void recorder.stop();
+			}
+		}
+	}, [recorder]);
+
+	const onStopRecording = useCallback(async () => {
+		try {
+			setRecordingState(RecorderState.Processing);
+			await recorder.stop();
+			isRecordingRef.current = false;
+			await resetAudioMode();
+
+			const saveEvent = await recordingToSaveData(recorder.uri);
+			onFileSaved(saveEvent);
+			onDismiss();
+		} catch (error) {
+			logger.error('Error saving recording:', error);
+			setError(`Save error: ${error}`);
+			setRecordingState(RecorderState.Error);
+		}
+	}, [onFileSaved, onDismiss, recorder]);
+
+	const onStartStopRecording = useCallback(async () => {
+		if (recordingState === RecorderState.Idle) {
+			await onStartRecording();
+		} else if (recordingState === RecorderState.Recording) {
+			await onStopRecording();
+		}
+	}, [recordingState, onStartRecording, onStopRecording]);
+
+	useEffect(() => () => {
+		if (isRecordingRef.current) {
+			isRecordingRef.current = false;
+
+			const stopRecorderOnCleanup = async () => {
+				try {
+					await recorder.stop();
+				} catch (error) {
+					logger.warn('Error stopping recorder during cleanup:', error);
+				}
+
+				await resetAudioMode();
+			};
+
+			void stopRecorderOnCleanup();
+		}
+	}, [recorder]);
+
+	return { onStartStopRecording, error, duration: recorderStatus.durationMillis, recordingState };
+};
+
+const AudioRecordingBanner: React.FC<Props> = props => {
+	const { recordingState, onStartStopRecording, duration, error } = useAudioRecorder(props.onFileSaved, props.onDismiss);
+
+	const onCancelPress = useCallback(async () => {
+		if (recordingState === RecorderState.Recording) {
+			const message = _('Cancelling will discard the recording. This cannot be undone. Are you sure you want to proceed?');
+			if (!await shim.showConfirmationDialog(message)) return;
+		}
+		props.onDismiss();
+	}, [recordingState, props.onDismiss]);
+
+	const startStopButtonLabel = recordingState === RecorderState.Idle ? _('Start recording') : _('Done');
+	const allowStartStop = recordingState === RecorderState.Idle || recordingState === RecorderState.Recording;
+	const actions = <>
+		<SecondaryButton onPress={onCancelPress}>{_('Cancel')}</SecondaryButton>
+		<PrimaryButton
+			disabled={!allowStartStop}
+			onPress={onStartStopRecording}
+			// Add additional accessibility information to make it clear that "Done" is
+			// associated with the voice recording banner:
+			accessibilityHint={recordingState === RecorderState.Recording ? _('Finishes recording') : undefined}
+		>{startStopButtonLabel}</PrimaryButton>
+	</>;
+
+	const renderDuration = () => {
+		if (recordingState !== RecorderState.Recording) return null;
+
+		const durationValue = formatMsToDurationCompat(duration);
+		return <Text
+			accessibilityLabel={_('Duration: %s', durationValue)}
+			accessibilityRole='timer'
+		>{durationValue}</Text>;
+	};
+
+	return <RecordingControls
+		recorderState={recordingState}
+		heading={recordingState === RecorderState.Recording ? _('Recording...') : _('Voice recorder')}
+		content={
+			recordingState === RecorderState.Idle
+				? _('Click "start" to attach a new voice memo to the note.')
+				: error
+		}
+		preview={renderDuration()}
+		actions={actions}
+	/>;
+};
+
+export default AudioRecordingBanner;

packages/app-mobile/components/voiceTyping/RecordingControls.tsx

@@ -0,0 +1,98 @@
+import * as React from 'react';
+import { View, StyleSheet } from 'react-native';
+import { ActivityIndicator, Icon, Surface, Text } from 'react-native-paper';
+import { IconSource } from 'react-native-paper/lib/typescript/components/Icon';
+import AccessibleView from '../accessibility/AccessibleView';
+import { RecorderState } from './types';
+
+interface Props {
+	recorderState: RecorderState;
+	heading: string;
+	content: React.ReactNode|string;
+	preview: React.ReactNode;
+	actions: React.ReactNode;
+}
+
+const styles = StyleSheet.create({
+	container: {
+		paddingHorizontal: 10,
+		width: 680,
+		flexShrink: 1,
+		maxWidth: '100%',
+		alignSelf: 'center',
+	},
+	contentWrapper: {
+		flexDirection: 'row',
+	},
+	iconWrapper: {
+		margin: 8,
+		marginTop: 16,
+	},
+	content: {
+		flexShrink: 1,
+		marginTop: 16,
+		marginHorizontal: 8,
+	},
+	actionContainer: {
+		flexDirection: 'row',
+		justifyContent: 'flex-end',
+		gap: 6,
+		marginBottom: 6,
+	},
+});
+
+const RecordingControls: React.FC<Props> = props => {
+	const renderIcon = () => {
+		const loadingIcon: IconSource = ({ size }: { size: number }) => {
+			return <ActivityIndicator animating={true} style={{ width: size, height: size }} />;
+		};
+		const components: Record<RecorderState, IconSource> = {
+			[RecorderState.Loading]: loadingIcon,
+			[RecorderState.Recording]: 'microphone',
+			[RecorderState.Idle]: 'microphone',
+			[RecorderState.Processing]: loadingIcon,
+			[RecorderState.Downloading]: loadingIcon,
+			[RecorderState.Error]: 'alert-circle-outline',
+		};
+
+		return components[props.recorderState];
+	};
+
+	return <Surface>
+		<View style={styles.container}>
+			<View style={styles.contentWrapper}>
+				<View style={styles.iconWrapper}>
+					<Icon source={renderIcon()} size={40}/>
+				</View>
+				<View style={styles.content}>
+					<AccessibleView
+						// Auto-focus
+						refocusCounter={1}
+						aria-live='polite'
+						role='heading'
+						testID='recording-controls-heading'
+					>
+						<Text variant='bodyMedium'>
+							{props.heading}
+						</Text>
+					</AccessibleView>
+					<Text
+						variant='bodyMedium'
+						// role="status" might fit better here. However, react-native
+						// doesn't seem to support it.
+						role='alert'
+						// Although on web, role=alert should imply aria-live=polite,
+						// this does not seem to be the case for React Native:
+						accessibilityLiveRegion='polite'
+					>{props.content}</Text>
+					{props.preview}
+				</View>
+			</View>
+			<View style={styles.actionContainer}>
+				{props.actions}
+			</View>
+		</View>
+	</Surface>;
+};
+
+export default RecordingControls;

packages/app-mobile/components/voiceTyping/SpeechToTextBanner.tsx

@@ -0,0 +1,205 @@
+import * as React from 'react';
+import { useState, useEffect, useCallback, useRef, useMemo } from 'react';
+import { Text, Button } from 'react-native-paper';
+import { _, languageName } from '@joplin/lib/locale';
+import useAsyncEffect, { AsyncEffectEvent } from '@joplin/lib/hooks/useAsyncEffect';
+import VoiceTyping, { OnTextCallback, VoiceTypingSession } from '../../services/voiceTyping/VoiceTyping';
+import { RecorderState } from './types';
+import RecordingControls from './RecordingControls';
+import { PrimaryButton } from '../buttons';
+import useQueuedAsyncEffect from '@joplin/lib/hooks/useQueuedAsyncEffect';
+import shim from '@joplin/lib/shim';
+
+interface Props {
+	locale: string;
+	onDismiss: ()=> void;
+	onText: (text: string)=> void;
+}
+
+interface UseVoiceTypingProps {
+	locale: string;
+	onSetPreview: OnTextCallback;
+	onText: OnTextCallback;
+}
+
+const useVoiceTyping = ({ locale, onSetPreview, onText }: UseVoiceTypingProps) => {
+	const [voiceTyping, setVoiceTyping] = useState<VoiceTypingSession>(null);
+	const [error, setError] = useState<Error|null>(null);
+	const [mustDownloadModel, setMustDownloadModel] = useState<boolean | null>(null);
+	const [stoppingSession, setIsStoppingSession] = useState(false);
+
+	const onTextRef = useRef(onText);
+	onTextRef.current = onText;
+	const onSetPreviewRef = useRef(onSetPreview);
+	onSetPreviewRef.current = onSetPreview;
+
+	const voiceTypingRef = useRef(voiceTyping);
+	voiceTypingRef.current = voiceTyping;
+
+	const builder = useMemo(() => {
+		return new VoiceTyping(locale);
+	}, [locale]);
+
+	const [redownloadCounter, setRedownloadCounter] = useState(0);
+
+	useQueuedAsyncEffect(async (event) => {
+		try {
+			// Reset the error: If starting voice typing again resolves the error, the error
+			// should be hidden (and voice typing should start).
+			setError(null);
+
+			await voiceTypingRef.current?.cancel();
+			onSetPreviewRef.current?.('');
+
+			const outdated = await builder.isDownloadedFromOutdatedUrl();
+			if (outdated) {
+				const allowOutdatedMessage = _('New model available\nA new voice typing model is available. Do you want to download it?');
+				const downloadNewModel = await shim.showConfirmationDialog(allowOutdatedMessage);
+				if (downloadNewModel) {
+					await onRequestRedownload();
+					return;
+				}
+			}
+
+			if (!await builder.isDownloaded()) {
+				if (event.cancelled) return;
+				await builder.download();
+			}
+			if (event.cancelled) return;
+
+			const voiceTyping = await builder.build({
+				onPreview: (text) => onSetPreviewRef.current(text),
+				onFinalize: (text) => onTextRef.current(text),
+			});
+			if (event.cancelled) return;
+			setVoiceTyping(voiceTyping);
+		} catch (error) {
+			setError(error);
+		} finally {
+			setMustDownloadModel(false);
+		}
+	}, [builder, redownloadCounter]);
+
+	useAsyncEffect(async (_event: AsyncEffectEvent) => {
+		setMustDownloadModel(!(await builder.isDownloaded()));
+	}, [builder]);
+
+	useEffect(() => () => {
+		void voiceTypingRef.current?.cancel();
+	}, []);
+
+	const onRequestRedownload = useCallback(async () => {
+		setIsStoppingSession(true);
+		await voiceTypingRef.current?.cancel();
+		await builder.clearDownloads();
+		setMustDownloadModel(true);
+		setIsStoppingSession(false);
+		setRedownloadCounter(value => value + 1);
+	}, [builder]);
+
+	return {
+		error, mustDownloadModel, stoppingSession, voiceTyping, onRequestRedownload,
+	};
+};
+
+const SpeechToTextComponent: React.FC<Props> = props => {
+	const [recorderState, setRecorderState] = useState<RecorderState>(RecorderState.Loading);
+	const [preview, setPreview] = useState<string>('');
+	const {
+		error: modelError,
+		mustDownloadModel,
+		voiceTyping,
+		stoppingSession,
+		onRequestRedownload,
+	} = useVoiceTyping({
+		locale: props.locale,
+		onSetPreview: setPreview,
+		onText: props.onText,
+	});
+
+	useEffect(() => {
+		if (modelError) {
+			setRecorderState(RecorderState.Error);
+		} else if (voiceTyping) {
+			setRecorderState(RecorderState.Recording);
+		}
+	}, [voiceTyping, modelError]);
+
+	useEffect(() => {
+		if (mustDownloadModel) {
+			setRecorderState(RecorderState.Downloading);
+		}
+	}, [mustDownloadModel]);
+
+	useEffect(() => {
+		if (stoppingSession) {
+			setRecorderState(RecorderState.Processing);
+		}
+	}, [stoppingSession]);
+
+	useEffect(() => {
+		if (recorderState === RecorderState.Recording) {
+			void voiceTyping.start();
+		}
+	}, [recorderState, voiceTyping, props.onText]);
+
+	const onDismiss = useCallback(async () => {
+		if (voiceTyping) {
+			setRecorderState(RecorderState.Processing);
+			await voiceTyping.stop();
+			setRecorderState(RecorderState.Idle);
+		}
+		props.onDismiss();
+	}, [voiceTyping, props.onDismiss]);
+
+	const renderContent = () => {
+		const components: Record<RecorderState, ()=> string> = {
+			[RecorderState.Loading]: () => _('Loading...'),
+			[RecorderState.Idle]: () => 'Waiting...', // Not used for now
+			[RecorderState.Recording]: () => _('Please record your voice...'),
+			[RecorderState.Processing]: () => (
+				stoppingSession ? _('Closing session...') : _('Converting speech to text...')
+			),
+			[RecorderState.Downloading]: () => _('Downloading %s language files...', languageName(props.locale)),
+			[RecorderState.Error]: () => _('Error: %s', modelError?.message),
+		};
+
+		return components[recorderState]();
+	};
+
+	const renderPreview = () => {
+		if (recorderState !== RecorderState.Recording) {
+			return null;
+		}
+		return <Text variant='labelSmall'>{preview}</Text>;
+	};
+
+	const reDownloadButton = <Button
+		// Usually, stoppingSession is true because the re-download button has
+		// just been pressed.
+		disabled={stoppingSession || recorderState === RecorderState.Downloading}
+		onPress={onRequestRedownload}
+	>
+		{_('Re-download model')}
+	</Button>;
+	const allowReDownload = recorderState === RecorderState.Error;
+
+	const actions = <>
+		{allowReDownload ? reDownloadButton : null}
+		<PrimaryButton
+			onPress={onDismiss}
+			disabled={recorderState === RecorderState.Processing}
+			accessibilityHint={_('Ends voice typing')}
+		>{_('Done')}</PrimaryButton>
+	</>;
+
+	return <RecordingControls
+		recorderState={recorderState}
+		heading={_('Voice typing...')}
+		content={renderContent()}
+		preview={renderPreview()}
+		actions={actions}
+	/>;
+};
+
+export default SpeechToTextComponent;

packages/app-mobile/components/voiceTyping/types.ts

@@ -0,0 +1,16 @@
+export interface OnFileEvent {
+	uri: string;
+	fileName: string;
+	type: string|undefined;
+}
+
+export type OnFileSavedCallback = (event: OnFileEvent)=> void;
+
+export enum RecorderState {
+	Loading = 1,
+	Recording = 2,
+	Processing = 3,
+	Error = 4,
+	Downloading = 5,
+	Idle = 6,
+}