import { envVar } from '@/env'; import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState, } from 'react'; import { createPortal } from 'react-dom'; import { useDropzone } from 'react-dropzone'; import { useTranslation } from 'react-i18next'; import { useDispatch, useSelector, useStore } from 'react-redux'; import endpoints from '../api/endpoints'; import userService from '../api/services/userService'; import DragFileUpload from '../assets/DragFileUpload.svg'; import SendArrow from '../assets/send.svg?react'; import SourceIcon from '../assets/source.svg'; import { addAttachment, removeAttachment, selectAttachments, updateAttachment, reorderAttachments, } from '../upload/uploadSlice'; import { ActiveState, Doc } from '../models/misc'; import { selectSelectedDocs, selectSelectedModel, selectSourceDocs, selectSttAvailable, selectToken, setSelectedDocs, } from '../preferences/preferenceSlice'; import type { RootState } from '../store'; import Upload from '../upload/Upload'; import { isTouchDevice } from '../utils/browserUtils'; import { Button } from './ui/button'; import { type MultiSelectPopoverItem } from './MultiSelectPopover'; import ToolIcon from './ToolIcon'; import { AttachFileButton, AttachmentChipList, MicButton, type RecordingState, SourcesTrigger, ToolsTrigger, } from './message-input'; import { useArmedSend } from './message-input/armedSend'; import { cannotReadAttachment } from './message-input/attachmentReadability'; import { handleAbort } from '../conversation/conversationSlice'; import { AUDIO_FILE_ACCEPT_ATTR, getFileExtension, parseUploadErrorMessage, parseUploadErrorsByIndex, partitionAttachmentFiles, } from '../constants/fileUpload'; import { UserToolType } from '../settings/types'; import { sourceItemId, toSourcePickerItems } from '../utils/sourceUtils'; import { isChatToolVisible } from '../utils/toolUtils'; const generateId = (): string => `${Date.now()}-${Math.random().toString(36).substring(2)}`; const LIVE_TRANSCRIPTION_TIMESLICE_MS = 1000; const LIVE_CAPTURE_SAMPLE_RATE = 16000; const LIVE_CAPTURE_MAX_BUFFER_SECONDS = 20; const LIVE_SILENCE_RMS_THRESHOLD = 0.015; const ENABLE_VOICE_INPUT = envVar('VITE_ENABLE_VOICE_INPUT') === 'true'; type AudioContextWindow = Window & typeof globalThis & { webkitAudioContext?: typeof AudioContext; }; type LegacyNavigator = Navigator & { getUserMedia?: ( constraints: MediaStreamConstraints, successCallback: (stream: MediaStream) => void, errorCallback: (error: DOMException) => void, ) => void; webkitGetUserMedia?: ( constraints: MediaStreamConstraints, successCallback: (stream: MediaStream) => void, errorCallback: (error: DOMException) => void, ) => void; mozGetUserMedia?: ( constraints: MediaStreamConstraints, successCallback: (stream: MediaStream) => void, errorCallback: (error: DOMException) => void, ) => void; }; type LiveAudioSnapshot = { blob: Blob; chunkIndex: number; isSilence: boolean; }; const getAudioContextConstructor = (): typeof AudioContext | null => { if (typeof window === 'undefined') { return null; } const audioWindow = window as AudioContextWindow; return audioWindow.AudioContext || audioWindow.webkitAudioContext || null; }; const getLegacyGetUserMedia = () => { if (typeof navigator === 'undefined') { return null; } const legacyNavigator = navigator as LegacyNavigator; return ( legacyNavigator.getUserMedia || legacyNavigator.webkitGetUserMedia || legacyNavigator.mozGetUserMedia || null ); }; const getVoiceInputSupportError = (): string | null => { if (typeof window === 'undefined' || typeof navigator === 'undefined') { return 'Voice input is unavailable right now.'; } if (!window.isSecureContext) { return 'Voice input requires a secure connection (HTTPS or localhost).'; } if (!navigator.mediaDevices?.getUserMedia && !getLegacyGetUserMedia()) { return 'Voice input is not available in this browser.'; } if (!getAudioContextConstructor()) { return 'Voice input requires Web Audio support in this browser.'; } return null; }; const getUserMediaStream = ( constraints: MediaStreamConstraints, ): Promise => { if (navigator.mediaDevices?.getUserMedia) { return navigator.mediaDevices.getUserMedia(constraints); } const legacyGetUserMedia = getLegacyGetUserMedia(); if (!legacyGetUserMedia) { return Promise.reject( new Error('Voice input is not available in this browser.'), ); } return new Promise((resolve, reject) => { legacyGetUserMedia.call(navigator, constraints, resolve, reject); }); }; const getVoiceInputErrorMessage = (error: unknown): string => { if (typeof window !== 'undefined' && !window.isSecureContext) { return 'Voice input requires a secure connection (HTTPS or localhost).'; } if (error instanceof DOMException) { switch (error.name) { case 'NotAllowedError': case 'PermissionDeniedError': case 'SecurityError': return 'Microphone access was blocked. Allow microphone permission and try again.'; case 'NotFoundError': case 'DevicesNotFoundError': return 'No microphone was found on this device.'; case 'NotReadableError': case 'TrackStartError': return 'The microphone is unavailable or already in use.'; case 'AbortError': return 'Microphone access was interrupted before recording started.'; default: break; } } if (error instanceof Error && error.message) { return error.message; } return 'Microphone access was denied.'; }; const downsampleFloat32Buffer = ( source: Float32Array, inputSampleRate: number, outputSampleRate: number, ): Float32Array => { if ( !source.length || inputSampleRate <= 0 || outputSampleRate <= 0 || inputSampleRate === outputSampleRate ) { return source; } if (outputSampleRate > inputSampleRate) { return source; } const ratio = inputSampleRate / outputSampleRate; const outputLength = Math.max(1, Math.round(source.length / ratio)); const output = new Float32Array(outputLength); let outputOffset = 0; let inputOffset = 0; while (outputOffset < output.length) { const nextInputOffset = Math.min( source.length, Math.round((outputOffset + 1) * ratio), ); let accumulator = 0; let count = 0; for (let index = inputOffset; index < nextInputOffset; index += 1) { accumulator += source[index]; count += 1; } output[outputOffset] = count > 0 ? accumulator / count : source[inputOffset]; outputOffset += 1; inputOffset = nextInputOffset; } return output; }; const concatenateFloat32Chunks = ( chunks: Float32Array[], totalLength: number, ): Float32Array => { const output = new Float32Array(totalLength); let offset = 0; chunks.forEach((chunk) => { output.set(chunk, offset); offset += chunk.length; }); return output; }; const encodeWavFromFloat32 = ( samples: Float32Array, sampleRate: number, ): Blob => { const bytesPerSample = 2; const blockAlign = bytesPerSample; const buffer = new ArrayBuffer(44 + samples.length * bytesPerSample); const view = new DataView(buffer); let offset = 0; const writeString = (value: string) => { for (let index = 0; index < value.length; index += 1) { view.setUint8(offset + index, value.charCodeAt(index)); } offset += value.length; }; writeString('RIFF'); view.setUint32(offset, 36 + samples.length * bytesPerSample, true); offset += 4; writeString('WAVE'); writeString('fmt '); view.setUint32(offset, 16, true); offset += 4; view.setUint16(offset, 1, true); offset += 2; view.setUint16(offset, 1, true); offset += 2; view.setUint32(offset, sampleRate, true); offset += 4; view.setUint32(offset, sampleRate * blockAlign, true); offset += 4; view.setUint16(offset, blockAlign, true); offset += 2; view.setUint16(offset, 16, true); offset += 2; writeString('data'); view.setUint32(offset, samples.length * bytesPerSample, true); offset += 4; for (let index = 0; index < samples.length; index += 1) { const clamped = Math.max(-1, Math.min(1, samples[index])); view.setInt16( offset, clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff, true, ); offset += 2; } return new Blob([buffer], { type: 'audio/wav' }); }; type MessageInputProps = { onSubmit: (text: string) => void; loading: boolean; showSourceButton?: boolean; showToolButton?: boolean; autoFocus?: boolean; // Opt-in: enable send with empty text when there are completed // attachments (used by doc-driven workflow runs). Normal chat leaves this // unset, preserving the text-required behavior. allowSendWithoutText?: boolean; // A question queued by a send path outside the composer (hero // suggestion cards) while attachments were still pending: seeds the // input and arms the send so the standard waiting banner takes over. queuedQuestion?: string | null; onQueuedQuestionConsumed?: () => void; }; export default function MessageInput({ onSubmit, loading, showSourceButton = true, showToolButton = true, autoFocus = true, allowSendWithoutText = false, queuedQuestion = null, onQueuedQuestionConsumed, }: MessageInputProps) { const { t } = useTranslation(); const [value, setValue] = useState(''); const inputRef = useRef(null); const voiceFileInputRef = useRef(null); const [isSourcesPopupOpen, setIsSourcesPopupOpen] = useState(false); const [isToolsPopupOpen, setIsToolsPopupOpen] = useState(false); const [userTools, setUserTools] = useState([]); const [toolsLoading, setToolsLoading] = useState(false); const [uploadModalState, setUploadModalState] = useState('INACTIVE'); const [handleDragActive, setHandleDragActive] = useState(false); const [recordingState, setRecordingState] = useState('idle'); const [voiceError, setVoiceError] = useState(null); const selectedDocs = useSelector(selectSelectedDocs); const sourceDocs = useSelector(selectSourceDocs); const token = useSelector(selectToken); const attachments = useSelector(selectAttachments); const selectedModel = useSelector(selectSelectedModel); const sttAvailable = useSelector(selectSttAvailable); const unreadableAttachmentIds = useMemo( () => new Set( attachments .filter((attachment) => cannotReadAttachment( attachment, selectedModel?.supported_attachment_types, ), ) .map((attachment) => attachment.id), ), [attachments, selectedModel], ); const dispatch = useDispatch(); const store = useStore(); const mediaStreamRef = useRef(null); const audioContextRef = useRef(null); const audioSourceNodeRef = useRef(null); const audioProcessorNodeRef = useRef(null); const audioSilenceGainRef = useRef(null); const snapshotIntervalRef = useRef(null); const pcmChunksRef = useRef([]); const totalBufferedSamplesRef = useRef(0); const totalCapturedSamplesRef = useRef(0); const lastSnapshotCapturedSamplesRef = useRef(0); const recentWindowRmsRef = useRef({ sumSquares: 0, sampleCount: 0 }); const liveSessionIdRef = useRef(null); const livePendingSnapshotRef = useRef(null); const liveChunkIndexRef = useRef(0); const liveUploadInFlightRef = useRef(false); const liveStopRequestedRef = useRef(false); const voiceBaseValueRef = useRef(''); const liveTranscriptRef = useRef(''); const isTouch = isTouchDevice(); const stopMediaStream = () => { mediaStreamRef.current?.getTracks().forEach((track) => track.stop()); mediaStreamRef.current = null; }; const stopAudioProcessing = () => { if (snapshotIntervalRef.current !== null) { window.clearInterval(snapshotIntervalRef.current); snapshotIntervalRef.current = null; } if (audioProcessorNodeRef.current) { audioProcessorNodeRef.current.onaudioprocess = null; audioProcessorNodeRef.current.disconnect(); audioProcessorNodeRef.current = null; } if (audioSourceNodeRef.current) { audioSourceNodeRef.current.disconnect(); audioSourceNodeRef.current = null; } if (audioSilenceGainRef.current) { audioSilenceGainRef.current.disconnect(); audioSilenceGainRef.current = null; } if (audioContextRef.current) { void audioContextRef.current.close().catch(() => undefined); audioContextRef.current = null; } stopMediaStream(); }; const resetLiveTranscriptionState = () => { pcmChunksRef.current = []; totalBufferedSamplesRef.current = 0; totalCapturedSamplesRef.current = 0; lastSnapshotCapturedSamplesRef.current = 0; recentWindowRmsRef.current = { sumSquares: 0, sampleCount: 0 }; liveSessionIdRef.current = null; livePendingSnapshotRef.current = null; liveChunkIndexRef.current = 0; liveUploadInFlightRef.current = false; liveStopRequestedRef.current = false; voiceBaseValueRef.current = ''; liveTranscriptRef.current = ''; }; useEffect(() => { return () => { stopAudioProcessing(); resetLiveTranscriptionState(); }; }, []); // Recover the race where attachment.* SSE arrives before the upload // XHR's onload sets ``attachmentId``: walk recentEvents and watchdog // the row so it can't stay stuck on 'processing'. Mirrors // Upload.tsx's ``trackTraining``. const trackAttachment = useCallback( (clientId: string, attachmentId: string) => { let handled = false; const check = () => { const state = store.getState(); const row = state.upload.attachments.find((a) => a.id === clientId); if (!row) return true; // removed by user; stop tracking if (row.status === 'completed' || row.status === 'failed') { handled = true; return true; } for (const event of state.notifications.recentEvents) { if (event.scope?.id !== attachmentId) continue; if (event.type === 'attachment.completed') { const payload = (event.payload || {}) as Record; const tokenCount = Number(payload.token_count); handled = true; dispatch( updateAttachment({ id: clientId, updates: { status: 'completed', progress: 100, ...(Number.isFinite(tokenCount) ? { token_count: tokenCount } : {}), ...(typeof payload.mime_type === 'string' ? { mimeType: payload.mime_type } : {}), ...(typeof payload.extraction_status === 'string' ? { extractionStatus: payload.extraction_status } : {}), }, }), ); return true; } if (event.type === 'attachment.failed') { handled = true; dispatch( updateAttachment({ id: clientId, updates: { status: 'failed' }, }), ); return true; } } return false; }; if (check()) return; const MAX_WAIT_MS = 5 * 60_000; let unsubscribe: (() => void) | null = null; const timer = window.setTimeout(() => { unsubscribe?.(); if (!handled) { handled = true; console.warn( 'trackAttachment: timed out waiting for terminal SSE', clientId, attachmentId, ); dispatch( updateAttachment({ id: clientId, updates: { status: 'failed' }, }), ); } }, MAX_WAIT_MS); unsubscribe = store.subscribe(() => { if (check()) { window.clearTimeout(timer); unsubscribe?.(); } }); }, [dispatch, store], ); const uploadFiles = useCallback( async (incomingFiles: File[]) => { if (!incomingFiles || incomingFiles.length === 0) return; // Run the server's own rule here, not just the input's `accept`: // mobile pickers ignore `accept`, and a file the server will refuse // should say so before it costs an upload. Surface the refusal as a // failed chip so the user sees why instead of a silent drop. const { supported, unsupported } = await partitionAttachmentFiles(incomingFiles); unsupported.forEach((file) => { dispatch( addAttachment({ id: generateId(), fileName: file.name, progress: 0, status: 'failed' as const, taskId: '', errorMessage: t('conversation.attachments.unsupportedType', { extension: getFileExtension(file.name) || '?', }), }), ); }); if (supported.length === 0) return; const files = supported; const apiHost = envVar('VITE_API_HOST'); if (files.length > 1) { const formData = new FormData(); const indexToUiId: Record = {}; files.forEach((file, i) => { formData.append('file', file); const uiId = generateId(); indexToUiId[i] = uiId; dispatch( addAttachment({ id: uiId, fileName: file.name, progress: 0, status: 'uploading' as const, taskId: '', }), ); }); const xhr = new XMLHttpRequest(); xhr.upload.addEventListener('progress', (event) => { if (event.lengthComputable) { const progress = Math.round((event.loaded / event.total) * 100); Object.values(indexToUiId).forEach((uiId) => dispatch( updateAttachment({ id: uiId, updates: { progress }, }), ), ); } }); xhr.onload = () => { const status = xhr.status; if (status === 200) { try { const response = JSON.parse(xhr.responseText); if (Array.isArray(response?.tasks)) { const tasks = response.tasks as Array<{ task_id?: string; filename?: string; attachment_id?: string; path?: string; upload_index?: number; }>; const errors = Array.isArray(response?.errors) ? (response.errors as Array<{ filename?: string; error?: string; upload_index?: number; }>) : []; const hasIndexedResults = tasks.some((task) => typeof task.upload_index === 'number') || errors.some( (errorItem) => typeof errorItem.upload_index === 'number', ); if (hasIndexedResults) { const tasksByIndex = new Map< number, (typeof tasks)[number] >(); const failedIndices = new Set(); tasks.forEach((task, taskOrderIndex) => { const uploadIndex = typeof task.upload_index === 'number' ? task.upload_index : taskOrderIndex; tasksByIndex.set(uploadIndex, task); }); const errorsByIndex = new Map(); errors.forEach((errorItem) => { if (typeof errorItem.upload_index === 'number') { failedIndices.add(errorItem.upload_index); errorsByIndex.set( errorItem.upload_index, errorItem.error, ); } }); files.forEach((_, index) => { const uiId = indexToUiId[index]; if (!uiId) return; const task = tasksByIndex.get(index); if (task?.task_id) { dispatch( updateAttachment({ id: uiId, updates: { taskId: task.task_id, // Stash the server's attachment id so SSE // ``attachment.*`` events can match this // row by ``scope.id`` and drive the // per-attachment push-fresh poll gate. attachmentId: task.attachment_id, status: 'processing', progress: 10, }, }), ); if (task.attachment_id) { trackAttachment(uiId, task.attachment_id); } return; } if (failedIndices.has(index)) { dispatch( updateAttachment({ id: uiId, updates: { status: 'failed', errorMessage: errorsByIndex.get(index), }, }), ); return; } dispatch( updateAttachment({ id: uiId, updates: { status: 'failed' }, }), ); }); } else { tasks.forEach((t, idx) => { const uiId = indexToUiId[idx]; if (!uiId) return; if (t?.task_id) { dispatch( updateAttachment({ id: uiId, updates: { taskId: t.task_id, attachmentId: t.attachment_id, status: 'processing', progress: 10, }, }), ); if (t.attachment_id) { trackAttachment(uiId, t.attachment_id); } } else { dispatch( updateAttachment({ id: uiId, updates: { status: 'failed' }, }), ); } }); if (tasks.length < files.length) { for (let i = tasks.length; i < files.length; i++) { const uiId = indexToUiId[i]; if (uiId) { dispatch( updateAttachment({ id: uiId, updates: { status: 'failed' }, }), ); } } } } } else if (response?.task_id) { if (files.length === 1) { const uiId = indexToUiId[0]; if (uiId) { dispatch( updateAttachment({ id: uiId, updates: { taskId: response.task_id, attachmentId: response.attachment_id, status: 'processing', progress: 10, }, }), ); if (response.attachment_id) { trackAttachment(uiId, response.attachment_id); } } } else { console.warn( 'Server returned a single task_id for multiple files. Update backend to return tasks[].', ); const firstUi = indexToUiId[0]; if (firstUi) { dispatch( updateAttachment({ id: firstUi, updates: { taskId: response.task_id, status: 'processing', progress: 10, }, }), ); } for (let i = 1; i < files.length; i++) { const uiId = indexToUiId[i]; if (uiId) { dispatch( updateAttachment({ id: uiId, updates: { status: 'failed' }, }), ); } } } } else { console.error('Unexpected upload response shape', response); Object.values(indexToUiId).forEach((id) => dispatch( updateAttachment({ id, updates: { status: 'failed' }, }), ), ); } } catch (err) { console.error( 'Failed to parse upload response', err, xhr.responseText, ); Object.values(indexToUiId).forEach((id) => dispatch( updateAttachment({ id, updates: { status: 'failed' }, }), ), ); } } else { console.error('Upload failed', status, xhr.responseText); // Each file gets its own reason where the server sent one; the // top-level message is the fallback, not the answer for all of // them — a batch can fail two files for two different reasons. const fallbackMessage = parseUploadErrorMessage(xhr.responseText); const errorsByIndex = parseUploadErrorsByIndex(xhr.responseText); Object.entries(indexToUiId).forEach(([index, id]) => dispatch( updateAttachment({ id, updates: { status: 'failed', errorMessage: errorsByIndex.get(Number(index)) ?? fallbackMessage, }, }), ), ); } }; xhr.onerror = () => { console.error('Upload network error'); Object.values(indexToUiId).forEach((id) => dispatch( updateAttachment({ id, updates: { status: 'failed' }, }), ), ); }; xhr.open('POST', `${apiHost}${endpoints.USER.STORE_ATTACHMENT}`); if (token) xhr.setRequestHeader('Authorization', `Bearer ${token}`); xhr.send(formData); return; } // Single-file path: upload each file individually (original repo behavior) files.forEach((file) => { const formData = new FormData(); formData.append('file', file); const xhr = new XMLHttpRequest(); const uniqueId = generateId(); const newAttachment = { id: uniqueId, fileName: file.name, progress: 0, status: 'uploading' as const, taskId: '', }; dispatch(addAttachment(newAttachment)); xhr.upload.addEventListener('progress', (event) => { if (event.lengthComputable) { const progress = Math.round((event.loaded / event.total) * 100); dispatch( updateAttachment({ id: uniqueId, updates: { progress }, }), ); } }); xhr.onload = () => { if (xhr.status === 200) { try { const response = JSON.parse(xhr.responseText); if (response.task_id) { dispatch( updateAttachment({ id: uniqueId, updates: { taskId: response.task_id, attachmentId: response.attachment_id, status: 'processing', progress: 10, }, }), ); if (response.attachment_id) { trackAttachment(uniqueId, response.attachment_id); } } else { // If backend returned tasks[] for single-file, handle gracefully: if ( Array.isArray(response?.tasks) && response.tasks[0]?.task_id ) { dispatch( updateAttachment({ id: uniqueId, updates: { taskId: response.tasks[0].task_id, attachmentId: response.tasks[0].attachment_id, status: 'processing', progress: 10, }, }), ); if (response.tasks[0].attachment_id) { trackAttachment(uniqueId, response.tasks[0].attachment_id); } } else { dispatch( updateAttachment({ id: uniqueId, updates: { status: 'failed' }, }), ); } } } catch (err) { console.error( 'Failed to parse upload response', err, xhr.responseText, ); dispatch( updateAttachment({ id: uniqueId, updates: { status: 'failed' }, }), ); } } else { dispatch( updateAttachment({ id: uniqueId, updates: { status: 'failed', errorMessage: parseUploadErrorMessage(xhr.responseText), }, }), ); } }; xhr.onerror = () => { dispatch( updateAttachment({ id: uniqueId, updates: { status: 'failed' }, }), ); }; xhr.open('POST', `${apiHost}${endpoints.USER.STORE_ATTACHMENT}`); if (token) xhr.setRequestHeader('Authorization', `Bearer ${token}`); xhr.send(formData); }); }, [dispatch, t, token, trackAttachment], ); const handleFileAttachment = (e: React.ChangeEvent) => { if (!e.target.files || e.target.files.length === 0) return; const files = Array.from(e.target.files); uploadFiles(files); // clear input so same file can be selected again e.target.value = ''; }; // Drag & drop via react-dropzone const onDrop = useCallback( (acceptedFiles: File[]) => { uploadFiles(acceptedFiles); setHandleDragActive(false); }, [uploadFiles], ); const { getRootProps, getInputProps } = useDropzone({ onDrop, noClick: true, noKeyboard: true, // The textarea below owns paste-to-attach via handlePaste. react-dropzone // added its own paste handling in v19.2 (on by default), which fires on the // root for pastes into any focused descendant - so both would upload the // same file. Leave paste to handlePaste. noPaste: true, multiple: true, onDragEnter: () => { setHandleDragActive(true); }, onDragLeave: () => { setHandleDragActive(false); }, maxSize: 25000000, // No `accept`: react-dropzone would drop a rejected file on the floor // with no feedback, and its mime matching disagrees with the server for // text files that have no parser (.py, .log). uploadFiles applies the // server's rule and reports what it refuses. }); const handleInput = useCallback(() => { if (!inputRef.current) return; if (window.innerWidth < 350) inputRef.current.style.height = 'auto'; else inputRef.current.style.height = '64px'; inputRef.current.style.height = `${Math.min( inputRef.current.scrollHeight, Math.round(window.innerHeight * 0.4), )}px`; }, []); const buildVoiceDraftValue = (baseText: string, transcript: string) => { const normalizedBaseText = baseText ?? ''; const normalizedTranscript = transcript.trim(); if (!normalizedTranscript) { return normalizedBaseText; } return normalizedBaseText.trim() ? `${normalizedBaseText}${ normalizedBaseText.endsWith('\n') ? '' : '\n' }${normalizedTranscript}` : normalizedTranscript; }; const applyLiveTranscript = (transcript: string) => { const normalizedTranscript = transcript.trim(); liveTranscriptRef.current = normalizedTranscript; setValue( buildVoiceDraftValue(voiceBaseValueRef.current, normalizedTranscript), ); }; const promptVoiceFileFallback = (message: string) => { setRecordingState('idle'); setVoiceError(`${message} Choose or record an audio file instead.`); setTimeout(() => { voiceFileInputRef.current?.click(); }, 0); }; const transcribeUploadedAudioFile = async (file: File) => { try { setVoiceError(null); setRecordingState('transcribing'); voiceBaseValueRef.current = value; liveTranscriptRef.current = ''; const response = await userService.transcribeAudio(file, token); const data = await response.json(); if (!response.ok || !data?.success) { throw new Error(data?.message || 'Failed to transcribe audio.'); } if (typeof data.text !== 'string' || !data.text.trim()) { throw new Error('No transcript was returned for this audio file.'); } applyLiveTranscript(data.text); setRecordingState('idle'); if (autoFocus) { setTimeout(() => { inputRef.current?.focus(); }, 0); } } catch (error) { console.error('Uploaded audio transcription failed', error); setRecordingState('error'); setVoiceError( error instanceof Error ? error.message : 'Failed to transcribe audio.', ); } }; const trimLivePcmBuffer = () => { const maxBufferedSamples = LIVE_CAPTURE_SAMPLE_RATE * LIVE_CAPTURE_MAX_BUFFER_SECONDS; while ( totalBufferedSamplesRef.current > maxBufferedSamples && pcmChunksRef.current.length > 1 ) { const removedChunk = pcmChunksRef.current.shift(); if (!removedChunk) { break; } totalBufferedSamplesRef.current -= removedChunk.length; } if ( totalBufferedSamplesRef.current > maxBufferedSamples && pcmChunksRef.current.length === 1 ) { const onlyChunk = pcmChunksRef.current[0]; if (!onlyChunk || onlyChunk.length <= maxBufferedSamples) { return; } const trimmedChunk = onlyChunk.slice( onlyChunk.length - maxBufferedSamples, ); pcmChunksRef.current = [trimmedChunk]; totalBufferedSamplesRef.current = trimmedChunk.length; } }; const cleanupLiveSession = async () => { const sessionId = liveSessionIdRef.current; if (!sessionId) { return; } liveSessionIdRef.current = null; try { await userService.finishLiveTranscription(sessionId, token); } catch { // Best-effort cleanup only. } }; const failLiveTranscription = async (message: string) => { console.error('Live audio transcription failed', message); stopAudioProcessing(); await cleanupLiveSession(); resetLiveTranscriptionState(); setRecordingState('error'); setVoiceError(message); }; const finalizeLiveTranscription = async () => { const sessionId = liveSessionIdRef.current; if (!sessionId) { resetLiveTranscriptionState(); setRecordingState('idle'); return; } try { const response = await userService.finishLiveTranscription( sessionId, token, ); const data = await response.json(); if (!response.ok || !data?.success) { throw new Error( data?.message || 'Failed to finalize live transcription.', ); } if (typeof data.text === 'string') { applyLiveTranscript(data.text); } setRecordingState('idle'); if (autoFocus) { setTimeout(() => { inputRef.current?.focus(); }, 0); } } catch (error) { console.error('Finalizing live audio transcription failed', error); setRecordingState('error'); setVoiceError( error instanceof Error ? error.message : 'Failed to finalize live transcription.', ); } finally { resetLiveTranscriptionState(); } }; const maybeFinalizeLiveTranscription = async () => { if ( !liveStopRequestedRef.current || liveUploadInFlightRef.current || livePendingSnapshotRef.current ) { return; } await finalizeLiveTranscription(); }; const processPendingLiveSnapshot = async () => { if (liveUploadInFlightRef.current) { return; } const nextSnapshot = livePendingSnapshotRef.current; const sessionId = liveSessionIdRef.current; if (!nextSnapshot || !sessionId) { await maybeFinalizeLiveTranscription(); return; } livePendingSnapshotRef.current = null; liveUploadInFlightRef.current = true; try { const file = new File( [nextSnapshot.blob], `voice-live-${nextSnapshot.chunkIndex}.wav`, { type: 'audio/wav', }, ); const response = await userService.transcribeLiveAudioChunk( sessionId, nextSnapshot.chunkIndex, file, token, nextSnapshot.isSilence, ); const data = await response.json(); if (!response.ok || !data?.success) { throw new Error(data?.message || 'Failed to transcribe audio.'); } if (typeof data.transcript_text === 'string') { applyLiveTranscript(data.transcript_text); } } catch (error) { await failLiveTranscription( error instanceof Error ? error.message : 'Failed to transcribe audio.', ); return; } finally { liveUploadInFlightRef.current = false; } if (livePendingSnapshotRef.current) { void processPendingLiveSnapshot(); return; } void maybeFinalizeLiveTranscription(); }; const queueCurrentLiveSnapshot = (forceSilence = false) => { if ( totalCapturedSamplesRef.current === lastSnapshotCapturedSamplesRef.current ) { return; } if (!pcmChunksRef.current.length || totalBufferedSamplesRef.current <= 0) { return; } const pcmSnapshot = concatenateFloat32Chunks( pcmChunksRef.current, totalBufferedSamplesRef.current, ); if (!pcmSnapshot.length) { return; } const { sumSquares, sampleCount } = recentWindowRmsRef.current; const averageRms = sampleCount > 0 ? Math.sqrt(sumSquares / sampleCount) : 0; const isSilence = forceSilence || averageRms < LIVE_SILENCE_RMS_THRESHOLD; recentWindowRmsRef.current = { sumSquares: 0, sampleCount: 0 }; lastSnapshotCapturedSamplesRef.current = totalCapturedSamplesRef.current; livePendingSnapshotRef.current = { blob: encodeWavFromFloat32(pcmSnapshot, LIVE_CAPTURE_SAMPLE_RATE), chunkIndex: liveChunkIndexRef.current, isSilence, }; liveChunkIndexRef.current += 1; void processPendingLiveSnapshot(); }; const handleVoiceInput = async () => { if (recordingState === 'transcribing') { return; } if (recordingState === 'recording') { setRecordingState('transcribing'); liveStopRequestedRef.current = true; stopAudioProcessing(); queueCurrentLiveSnapshot(); void maybeFinalizeLiveTranscription(); return; } const voiceInputSupportError = getVoiceInputSupportError(); if (voiceInputSupportError) { promptVoiceFileFallback(voiceInputSupportError); return; } const AudioContextConstructor = getAudioContextConstructor(); if (!AudioContextConstructor) { setRecordingState('error'); setVoiceError('Voice input requires Web Audio support in this browser.'); return; } let stream: MediaStream | null = null; try { setVoiceError(null); stream = await getUserMediaStream({ audio: true }); } catch (error) { promptVoiceFileFallback(getVoiceInputErrorMessage(error)); return; } try { const liveStartResponse = await userService.startLiveTranscription(token); const liveStartData = await liveStartResponse.json(); if (!liveStartResponse.ok || !liveStartData?.success) { throw new Error( liveStartData?.message || 'Failed to start live transcription.', ); } const audioContext = new AudioContextConstructor(); await audioContext.resume().catch(() => undefined); const sourceNode = audioContext.createMediaStreamSource(stream); const processorNode = audioContext.createScriptProcessor(4096, 1, 1); const silenceGain = audioContext.createGain(); silenceGain.gain.value = 0; pcmChunksRef.current = []; totalBufferedSamplesRef.current = 0; totalCapturedSamplesRef.current = 0; lastSnapshotCapturedSamplesRef.current = 0; recentWindowRmsRef.current = { sumSquares: 0, sampleCount: 0 }; liveSessionIdRef.current = liveStartData.session_id; livePendingSnapshotRef.current = null; liveChunkIndexRef.current = 0; liveUploadInFlightRef.current = false; liveStopRequestedRef.current = false; voiceBaseValueRef.current = value; liveTranscriptRef.current = ''; applyLiveTranscript(''); processorNode.onaudioprocess = (event: AudioProcessingEvent) => { const inputData = event.inputBuffer.getChannelData(0); if (!inputData.length) { return; } const capturedChunk = new Float32Array(inputData.length); capturedChunk.set(inputData); const downsampledChunk = downsampleFloat32Buffer( capturedChunk, audioContext.sampleRate, LIVE_CAPTURE_SAMPLE_RATE, ); if (!downsampledChunk.length) { return; } pcmChunksRef.current.push(downsampledChunk); totalBufferedSamplesRef.current += downsampledChunk.length; totalCapturedSamplesRef.current += downsampledChunk.length; let sumSquares = 0; for (let index = 0; index < downsampledChunk.length; index += 1) { const sample = downsampledChunk[index]; sumSquares += sample * sample; } recentWindowRmsRef.current.sumSquares += sumSquares; recentWindowRmsRef.current.sampleCount += downsampledChunk.length; trimLivePcmBuffer(); }; sourceNode.connect(processorNode); processorNode.connect(silenceGain); silenceGain.connect(audioContext.destination); mediaStreamRef.current = stream; audioContextRef.current = audioContext; audioSourceNodeRef.current = sourceNode; audioProcessorNodeRef.current = processorNode; audioSilenceGainRef.current = silenceGain; snapshotIntervalRef.current = window.setInterval(() => { if (!liveStopRequestedRef.current) { queueCurrentLiveSnapshot(); } }, LIVE_TRANSCRIPTION_TIMESLICE_MS); setRecordingState('recording'); } catch (error) { console.error('Live voice transcription failed', error); stream?.getTracks().forEach((track) => track.stop()); stopAudioProcessing(); await cleanupLiveSession(); resetLiveTranscriptionState(); setRecordingState('error'); setVoiceError( error instanceof Error ? error.message : 'Failed to start live transcription.', ); } }; const isMountedRef = useRef(true); useEffect(() => { isMountedRef.current = true; return () => { isMountedRef.current = false; }; }, []); useLayoutEffect(() => { handleInput(); }, [value, handleInput]); useEffect(() => { window.addEventListener('resize', handleInput); return () => window.removeEventListener('resize', handleInput); }, [handleInput]); const handleChange = (e: React.ChangeEvent) => { setValue(e.target.value); }; const handleKeyDown = (e: React.KeyboardEvent) => { if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); handleSubmit(); } }; const handlePaste = (e: React.ClipboardEvent) => { const clipboardItems = e.clipboardData?.items; const files: File[] = []; if (!clipboardItems) return; for (let i = 0; i < clipboardItems.length; i++) { const item = clipboardItems[i]; if (item.kind === 'file') { const file = item.getAsFile(); if (file) { files.push(file); } } } if (files.length > 0) { // Prevent weird binary stuff from being pasted as text e.preventDefault(); uploadFiles(files); } }; const handleVoiceFileAttachment = ( e: React.ChangeEvent, ) => { const file = e.target.files?.[0]; e.target.value = ''; if (!file) { return; } void transcribeUploadedAudioFile(file); }; const handlePostDocumentSelect = (_docs: Doc[] | null) => { // Hook point for downstream side-effects after a source is toggled. void _docs; }; const sourceItems: MultiSelectPopoverItem[] = useMemo( () => toSourcePickerItems( sourceDocs, { own: t('conversation.sources.groupOwn'), team: t('conversation.sources.groupTeam'), }, SourceIcon, ), [sourceDocs, t], ); const selectedSourceIds = ( selectedDocs && Array.isArray(selectedDocs) ? selectedDocs : [] ).map((doc) => sourceItemId(doc)); const handleToggleSource = (id: string) => { if (!sourceDocs) return; const current = Array.isArray(selectedDocs) ? selectedDocs : []; const matched = current.find((doc) => sourceItemId(doc) === id); let updated: Doc[]; if (matched) { updated = current.filter((doc) => sourceItemId(doc) !== id); } else { const incoming = sourceDocs.find((doc) => sourceItemId(doc) === id); if (!incoming) return; updated = [...current, incoming]; } dispatch(setSelectedDocs(updated.length > 0 ? updated : [])); handlePostDocumentSelect(updated.length > 0 ? updated : null); }; const fetchUserTools = useCallback(() => { setToolsLoading(true); userService .getUserTools(token) .then((res) => res.json()) .then((data) => { const filtered = (data.tools || []).filter(isChatToolVisible); setUserTools(filtered); }) .catch((error) => { console.error('Error fetching tools:', error); }) .finally(() => setToolsLoading(false)); }, [token]); useEffect(() => { if (isToolsPopupOpen) fetchUserTools(); }, [isToolsPopupOpen, fetchUserTools]); const toolItems: MultiSelectPopoverItem[] = userTools.map((tool) => ({ id: tool.id, label: tool.customName || tool.displayName, icon: , })); const selectedToolIds = userTools .filter((tool) => tool.status) .map((tool) => tool.id); const handleToggleTool = (id: string) => { const tool = userTools.find((t) => t.id === id); if (!tool) return; const newStatus = !tool.status; userService .updateToolStatus({ id, status: newStatus }, token) .then(() => { setUserTools((prev) => prev.map((t) => (t.id === id ? { ...t, status: newStatus } : t)), ); }) .catch((error) => { console.error('Failed to update tool status:', error); }); }; const handleUploadClick = () => { setUploadModalState('ACTIVE'); setIsSourcesPopupOpen(false); }; // When ``allowSendWithoutText`` is set, an attachment-only submit is // permitted as long as at least one attachment exists; a still-pending // one arms the send instead of submitting (see handleSubmit). const hasSubmittableContent = Boolean(value.trim()) || (allowSendWithoutText && attachments.length > 0); const canSubmit = hasSubmittableContent && !loading && recordingState !== 'recording' && recordingState !== 'transcribing'; const submitNow = () => { onSubmit(value); setValue(''); if (isTouch) { inputRef.current?.blur(); } else if (autoFocus) { setTimeout(() => { if (isMountedRef.current) { inputRef.current?.focus(); } }, 0); } }; const { armed: sendArmed, readiness: sendReadiness, arm: armSend, cancel: cancelArmedSend, } = useArmedSend({ attachments, onFlush: submitNow }); // Adopt a question queued outside the composer: seed the input, arm, // and hand the wait to the standard banner. If the attachments already // resolved by the time this runs, the armed-send effect flushes at once. useEffect(() => { if (queuedQuestion == null || queuedQuestion === '') return; setValue(queuedQuestion); armSend(); onQueuedQuestionConsumed?.(); }, [queuedQuestion]); const handleSubmit = () => { if (!canSubmit) return; // Attachments still uploading/parsing (or failed) must never be // silently dropped from the payload: hold the send in the composer // until every attachment resolves, then flush automatically. if (sendReadiness.state !== 'ready') { armSend(); return; } submitNow(); }; const handleCancel = () => { handleAbort(); }; const [draggingId, setDraggingId] = useState(null); const findIndexById = (id: string) => attachments.findIndex((a) => a.id === id); const handleDragStart = (e: React.DragEvent, id: string) => { setDraggingId(id); try { e.dataTransfer.setData('text/plain', id); e.dataTransfer.effectAllowed = 'move'; } catch { // ignore } }; const handleDragOver = (e: React.DragEvent) => { e.preventDefault(); e.dataTransfer.dropEffect = 'move'; }; const handleDropOn = (e: React.DragEvent, targetId: string) => { e.preventDefault(); const sourceId = e.dataTransfer.getData('text/plain'); if (!sourceId || sourceId === targetId) return; const sourceIndex = findIndexById(sourceId); const destIndex = findIndexById(targetId); if (sourceIndex === -1 || destIndex === -1) return; dispatch(reorderAttachments({ sourceIndex, destinationIndex: destIndex })); setDraggingId(null); }; return (
{/* react-dropzone input (for drag/drop) */} {/* translate="no": keep Chrome's page translator out of the composer — it rewrites text nodes into wrappers and React loses the controls (dead Attach button, see AttachFileButton). */}
dispatch(removeAttachment(id))} onDragStart={handleDragStart} onDragOver={handleDragOver} onDropOn={handleDropOn} unreadableIds={unreadableAttachmentIds} modelName={selectedModel?.display_name} /> {sendArmed && sendReadiness.state === 'waiting' && (
{t('conversation.attachments.waitingToSend', { count: sendReadiness.pendingCount, })}
)} {sendArmed && sendReadiness.state === 'blocked' && (
{t('conversation.attachments.sendBlockedByFailed', { names: sendReadiness.failedNames.join(', '), })}
)} {voiceError && (
{voiceError}
)}