mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-03 11:11:58 +00:00
Alex
7da46c2bea
- Ship tiktoken's cl100k_base inside the package and build the encoding from it, so token counting never downloads anything. - Default EMBEDDINGS_CACHE_DIR to <data home>/models instead of FastEmbed's temp dir, and read tokenizer.json and repo metadata from that cache, so a model downloads once and survives reboots. - TTS_PROVIDER=none and STT_PROVIDER=none switch the speech features off: the endpoints return 404, audio files fail to ingest with a clear message, /api/config reports tts_available/stt_available, and the UI hides the Speak and microphone buttons. - Drop the Google Fonts Roboto import from the web UI. - prefetch-models fills the cache the app reads; verify-offline checks the packaged encoding. - Docs: new Air-Gapped Deployment guide, settings and cache notes.
1851 lines
58 KiB
TypeScript
1851 lines
58 KiB
TypeScript
import { envVar } from '@/env';
|
|
import {
|
|
useCallback,
|
|
useEffect,
|
|
useLayoutEffect,
|
|
useMemo,
|
|
useRef,
|
|
useState,
|
|
} from 'react';
|
|
import { createPortal } from 'react-dom';
|
|
import { useDropzone } from 'react-dropzone';
|
|
import { useTranslation } from 'react-i18next';
|
|
import { useDispatch, useSelector, useStore } from 'react-redux';
|
|
|
|
import endpoints from '../api/endpoints';
|
|
import userService from '../api/services/userService';
|
|
import DragFileUpload from '../assets/DragFileUpload.svg';
|
|
import SendArrow from '../assets/send.svg?react';
|
|
import SourceIcon from '../assets/source.svg';
|
|
import {
|
|
addAttachment,
|
|
removeAttachment,
|
|
selectAttachments,
|
|
updateAttachment,
|
|
reorderAttachments,
|
|
} from '../upload/uploadSlice';
|
|
|
|
import { ActiveState, Doc } from '../models/misc';
|
|
import {
|
|
selectSelectedDocs,
|
|
selectSelectedModel,
|
|
selectSourceDocs,
|
|
selectSttAvailable,
|
|
selectToken,
|
|
setSelectedDocs,
|
|
} from '../preferences/preferenceSlice';
|
|
import type { RootState } from '../store';
|
|
import Upload from '../upload/Upload';
|
|
import { isTouchDevice } from '../utils/browserUtils';
|
|
import { Button } from './ui/button';
|
|
import { type MultiSelectPopoverItem } from './MultiSelectPopover';
|
|
import ToolIcon from './ToolIcon';
|
|
import {
|
|
AttachFileButton,
|
|
AttachmentChipList,
|
|
MicButton,
|
|
type RecordingState,
|
|
SourcesTrigger,
|
|
ToolsTrigger,
|
|
} from './message-input';
|
|
import { useArmedSend } from './message-input/armedSend';
|
|
import { cannotReadAttachment } from './message-input/attachmentReadability';
|
|
import { handleAbort } from '../conversation/conversationSlice';
|
|
import {
|
|
AUDIO_FILE_ACCEPT_ATTR,
|
|
getFileExtension,
|
|
parseUploadErrorMessage,
|
|
parseUploadErrorsByIndex,
|
|
partitionAttachmentFiles,
|
|
} from '../constants/fileUpload';
|
|
import { UserToolType } from '../settings/types';
|
|
import { sourceItemId, toSourcePickerItems } from '../utils/sourceUtils';
|
|
import { isChatToolVisible } from '../utils/toolUtils';
|
|
|
|
const generateId = (): string =>
|
|
`${Date.now()}-${Math.random().toString(36).substring(2)}`;
|
|
|
|
const LIVE_TRANSCRIPTION_TIMESLICE_MS = 1000;
|
|
const LIVE_CAPTURE_SAMPLE_RATE = 16000;
|
|
const LIVE_CAPTURE_MAX_BUFFER_SECONDS = 20;
|
|
const LIVE_SILENCE_RMS_THRESHOLD = 0.015;
|
|
const ENABLE_VOICE_INPUT = envVar('VITE_ENABLE_VOICE_INPUT') === 'true';
|
|
|
|
type AudioContextWindow = Window &
|
|
typeof globalThis & {
|
|
webkitAudioContext?: typeof AudioContext;
|
|
};
|
|
|
|
type LegacyNavigator = Navigator & {
|
|
getUserMedia?: (
|
|
constraints: MediaStreamConstraints,
|
|
successCallback: (stream: MediaStream) => void,
|
|
errorCallback: (error: DOMException) => void,
|
|
) => void;
|
|
webkitGetUserMedia?: (
|
|
constraints: MediaStreamConstraints,
|
|
successCallback: (stream: MediaStream) => void,
|
|
errorCallback: (error: DOMException) => void,
|
|
) => void;
|
|
mozGetUserMedia?: (
|
|
constraints: MediaStreamConstraints,
|
|
successCallback: (stream: MediaStream) => void,
|
|
errorCallback: (error: DOMException) => void,
|
|
) => void;
|
|
};
|
|
|
|
type LiveAudioSnapshot = {
|
|
blob: Blob;
|
|
chunkIndex: number;
|
|
isSilence: boolean;
|
|
};
|
|
|
|
const getAudioContextConstructor = (): typeof AudioContext | null => {
|
|
if (typeof window === 'undefined') {
|
|
return null;
|
|
}
|
|
|
|
const audioWindow = window as AudioContextWindow;
|
|
return audioWindow.AudioContext || audioWindow.webkitAudioContext || null;
|
|
};
|
|
|
|
const getLegacyGetUserMedia = () => {
|
|
if (typeof navigator === 'undefined') {
|
|
return null;
|
|
}
|
|
|
|
const legacyNavigator = navigator as LegacyNavigator;
|
|
return (
|
|
legacyNavigator.getUserMedia ||
|
|
legacyNavigator.webkitGetUserMedia ||
|
|
legacyNavigator.mozGetUserMedia ||
|
|
null
|
|
);
|
|
};
|
|
|
|
const getVoiceInputSupportError = (): string | null => {
|
|
if (typeof window === 'undefined' || typeof navigator === 'undefined') {
|
|
return 'Voice input is unavailable right now.';
|
|
}
|
|
|
|
if (!window.isSecureContext) {
|
|
return 'Voice input requires a secure connection (HTTPS or localhost).';
|
|
}
|
|
|
|
if (!navigator.mediaDevices?.getUserMedia && !getLegacyGetUserMedia()) {
|
|
return 'Voice input is not available in this browser.';
|
|
}
|
|
|
|
if (!getAudioContextConstructor()) {
|
|
return 'Voice input requires Web Audio support in this browser.';
|
|
}
|
|
|
|
return null;
|
|
};
|
|
|
|
const getUserMediaStream = (
|
|
constraints: MediaStreamConstraints,
|
|
): Promise<MediaStream> => {
|
|
if (navigator.mediaDevices?.getUserMedia) {
|
|
return navigator.mediaDevices.getUserMedia(constraints);
|
|
}
|
|
|
|
const legacyGetUserMedia = getLegacyGetUserMedia();
|
|
if (!legacyGetUserMedia) {
|
|
return Promise.reject(
|
|
new Error('Voice input is not available in this browser.'),
|
|
);
|
|
}
|
|
|
|
return new Promise((resolve, reject) => {
|
|
legacyGetUserMedia.call(navigator, constraints, resolve, reject);
|
|
});
|
|
};
|
|
|
|
const getVoiceInputErrorMessage = (error: unknown): string => {
|
|
if (typeof window !== 'undefined' && !window.isSecureContext) {
|
|
return 'Voice input requires a secure connection (HTTPS or localhost).';
|
|
}
|
|
|
|
if (error instanceof DOMException) {
|
|
switch (error.name) {
|
|
case 'NotAllowedError':
|
|
case 'PermissionDeniedError':
|
|
case 'SecurityError':
|
|
return 'Microphone access was blocked. Allow microphone permission and try again.';
|
|
case 'NotFoundError':
|
|
case 'DevicesNotFoundError':
|
|
return 'No microphone was found on this device.';
|
|
case 'NotReadableError':
|
|
case 'TrackStartError':
|
|
return 'The microphone is unavailable or already in use.';
|
|
case 'AbortError':
|
|
return 'Microphone access was interrupted before recording started.';
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (error instanceof Error && error.message) {
|
|
return error.message;
|
|
}
|
|
|
|
return 'Microphone access was denied.';
|
|
};
|
|
|
|
const downsampleFloat32Buffer = (
|
|
source: Float32Array,
|
|
inputSampleRate: number,
|
|
outputSampleRate: number,
|
|
): Float32Array => {
|
|
if (
|
|
!source.length ||
|
|
inputSampleRate <= 0 ||
|
|
outputSampleRate <= 0 ||
|
|
inputSampleRate === outputSampleRate
|
|
) {
|
|
return source;
|
|
}
|
|
|
|
if (outputSampleRate > inputSampleRate) {
|
|
return source;
|
|
}
|
|
|
|
const ratio = inputSampleRate / outputSampleRate;
|
|
const outputLength = Math.max(1, Math.round(source.length / ratio));
|
|
const output = new Float32Array(outputLength);
|
|
|
|
let outputOffset = 0;
|
|
let inputOffset = 0;
|
|
while (outputOffset < output.length) {
|
|
const nextInputOffset = Math.min(
|
|
source.length,
|
|
Math.round((outputOffset + 1) * ratio),
|
|
);
|
|
let accumulator = 0;
|
|
let count = 0;
|
|
for (let index = inputOffset; index < nextInputOffset; index += 1) {
|
|
accumulator += source[index];
|
|
count += 1;
|
|
}
|
|
output[outputOffset] =
|
|
count > 0 ? accumulator / count : source[inputOffset];
|
|
outputOffset += 1;
|
|
inputOffset = nextInputOffset;
|
|
}
|
|
|
|
return output;
|
|
};
|
|
|
|
const concatenateFloat32Chunks = (
|
|
chunks: Float32Array[],
|
|
totalLength: number,
|
|
): Float32Array => {
|
|
const output = new Float32Array(totalLength);
|
|
let offset = 0;
|
|
chunks.forEach((chunk) => {
|
|
output.set(chunk, offset);
|
|
offset += chunk.length;
|
|
});
|
|
return output;
|
|
};
|
|
|
|
const encodeWavFromFloat32 = (
|
|
samples: Float32Array,
|
|
sampleRate: number,
|
|
): Blob => {
|
|
const bytesPerSample = 2;
|
|
const blockAlign = bytesPerSample;
|
|
const buffer = new ArrayBuffer(44 + samples.length * bytesPerSample);
|
|
const view = new DataView(buffer);
|
|
let offset = 0;
|
|
|
|
const writeString = (value: string) => {
|
|
for (let index = 0; index < value.length; index += 1) {
|
|
view.setUint8(offset + index, value.charCodeAt(index));
|
|
}
|
|
offset += value.length;
|
|
};
|
|
|
|
writeString('RIFF');
|
|
view.setUint32(offset, 36 + samples.length * bytesPerSample, true);
|
|
offset += 4;
|
|
writeString('WAVE');
|
|
writeString('fmt ');
|
|
view.setUint32(offset, 16, true);
|
|
offset += 4;
|
|
view.setUint16(offset, 1, true);
|
|
offset += 2;
|
|
view.setUint16(offset, 1, true);
|
|
offset += 2;
|
|
view.setUint32(offset, sampleRate, true);
|
|
offset += 4;
|
|
view.setUint32(offset, sampleRate * blockAlign, true);
|
|
offset += 4;
|
|
view.setUint16(offset, blockAlign, true);
|
|
offset += 2;
|
|
view.setUint16(offset, 16, true);
|
|
offset += 2;
|
|
writeString('data');
|
|
view.setUint32(offset, samples.length * bytesPerSample, true);
|
|
offset += 4;
|
|
|
|
for (let index = 0; index < samples.length; index += 1) {
|
|
const clamped = Math.max(-1, Math.min(1, samples[index]));
|
|
view.setInt16(
|
|
offset,
|
|
clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff,
|
|
true,
|
|
);
|
|
offset += 2;
|
|
}
|
|
|
|
return new Blob([buffer], { type: 'audio/wav' });
|
|
};
|
|
|
|
type MessageInputProps = {
|
|
onSubmit: (text: string) => void;
|
|
loading: boolean;
|
|
showSourceButton?: boolean;
|
|
showToolButton?: boolean;
|
|
autoFocus?: boolean;
|
|
// Opt-in: enable send with empty text when there are completed
|
|
// attachments (used by doc-driven workflow runs). Normal chat leaves this
|
|
// unset, preserving the text-required behavior.
|
|
allowSendWithoutText?: boolean;
|
|
// A question queued by a send path outside the composer (hero
|
|
// suggestion cards) while attachments were still pending: seeds the
|
|
// input and arms the send so the standard waiting banner takes over.
|
|
queuedQuestion?: string | null;
|
|
onQueuedQuestionConsumed?: () => void;
|
|
};
|
|
|
|
export default function MessageInput({
|
|
onSubmit,
|
|
loading,
|
|
showSourceButton = true,
|
|
showToolButton = true,
|
|
autoFocus = true,
|
|
allowSendWithoutText = false,
|
|
queuedQuestion = null,
|
|
onQueuedQuestionConsumed,
|
|
}: MessageInputProps) {
|
|
const { t } = useTranslation();
|
|
const [value, setValue] = useState('');
|
|
const inputRef = useRef<HTMLTextAreaElement>(null);
|
|
const voiceFileInputRef = useRef<HTMLInputElement>(null);
|
|
const [isSourcesPopupOpen, setIsSourcesPopupOpen] = useState(false);
|
|
const [isToolsPopupOpen, setIsToolsPopupOpen] = useState(false);
|
|
const [userTools, setUserTools] = useState<UserToolType[]>([]);
|
|
const [toolsLoading, setToolsLoading] = useState(false);
|
|
const [uploadModalState, setUploadModalState] =
|
|
useState<ActiveState>('INACTIVE');
|
|
const [handleDragActive, setHandleDragActive] = useState<boolean>(false);
|
|
const [recordingState, setRecordingState] = useState<RecordingState>('idle');
|
|
const [voiceError, setVoiceError] = useState<string | null>(null);
|
|
|
|
const selectedDocs = useSelector(selectSelectedDocs);
|
|
const sourceDocs = useSelector(selectSourceDocs);
|
|
const token = useSelector(selectToken);
|
|
const attachments = useSelector(selectAttachments);
|
|
const selectedModel = useSelector(selectSelectedModel);
|
|
const sttAvailable = useSelector(selectSttAvailable);
|
|
const unreadableAttachmentIds = useMemo(
|
|
() =>
|
|
new Set(
|
|
attachments
|
|
.filter((attachment) =>
|
|
cannotReadAttachment(
|
|
attachment,
|
|
selectedModel?.supported_attachment_types,
|
|
),
|
|
)
|
|
.map((attachment) => attachment.id),
|
|
),
|
|
[attachments, selectedModel],
|
|
);
|
|
|
|
const dispatch = useDispatch();
|
|
const store = useStore<RootState>();
|
|
const mediaStreamRef = useRef<MediaStream | null>(null);
|
|
const audioContextRef = useRef<AudioContext | null>(null);
|
|
const audioSourceNodeRef = useRef<MediaStreamAudioSourceNode | null>(null);
|
|
const audioProcessorNodeRef = useRef<ScriptProcessorNode | null>(null);
|
|
const audioSilenceGainRef = useRef<GainNode | null>(null);
|
|
const snapshotIntervalRef = useRef<number | null>(null);
|
|
const pcmChunksRef = useRef<Float32Array[]>([]);
|
|
const totalBufferedSamplesRef = useRef(0);
|
|
const totalCapturedSamplesRef = useRef(0);
|
|
const lastSnapshotCapturedSamplesRef = useRef(0);
|
|
const recentWindowRmsRef = useRef({ sumSquares: 0, sampleCount: 0 });
|
|
const liveSessionIdRef = useRef<string | null>(null);
|
|
const livePendingSnapshotRef = useRef<LiveAudioSnapshot | null>(null);
|
|
const liveChunkIndexRef = useRef(0);
|
|
const liveUploadInFlightRef = useRef(false);
|
|
const liveStopRequestedRef = useRef(false);
|
|
const voiceBaseValueRef = useRef('');
|
|
const liveTranscriptRef = useRef('');
|
|
|
|
const isTouch = isTouchDevice();
|
|
|
|
const stopMediaStream = () => {
|
|
mediaStreamRef.current?.getTracks().forEach((track) => track.stop());
|
|
mediaStreamRef.current = null;
|
|
};
|
|
|
|
const stopAudioProcessing = () => {
|
|
if (snapshotIntervalRef.current !== null) {
|
|
window.clearInterval(snapshotIntervalRef.current);
|
|
snapshotIntervalRef.current = null;
|
|
}
|
|
|
|
if (audioProcessorNodeRef.current) {
|
|
audioProcessorNodeRef.current.onaudioprocess = null;
|
|
audioProcessorNodeRef.current.disconnect();
|
|
audioProcessorNodeRef.current = null;
|
|
}
|
|
if (audioSourceNodeRef.current) {
|
|
audioSourceNodeRef.current.disconnect();
|
|
audioSourceNodeRef.current = null;
|
|
}
|
|
if (audioSilenceGainRef.current) {
|
|
audioSilenceGainRef.current.disconnect();
|
|
audioSilenceGainRef.current = null;
|
|
}
|
|
if (audioContextRef.current) {
|
|
void audioContextRef.current.close().catch(() => undefined);
|
|
audioContextRef.current = null;
|
|
}
|
|
stopMediaStream();
|
|
};
|
|
|
|
const resetLiveTranscriptionState = () => {
|
|
pcmChunksRef.current = [];
|
|
totalBufferedSamplesRef.current = 0;
|
|
totalCapturedSamplesRef.current = 0;
|
|
lastSnapshotCapturedSamplesRef.current = 0;
|
|
recentWindowRmsRef.current = { sumSquares: 0, sampleCount: 0 };
|
|
liveSessionIdRef.current = null;
|
|
livePendingSnapshotRef.current = null;
|
|
liveChunkIndexRef.current = 0;
|
|
liveUploadInFlightRef.current = false;
|
|
liveStopRequestedRef.current = false;
|
|
voiceBaseValueRef.current = '';
|
|
liveTranscriptRef.current = '';
|
|
};
|
|
|
|
useEffect(() => {
|
|
return () => {
|
|
stopAudioProcessing();
|
|
resetLiveTranscriptionState();
|
|
};
|
|
}, []);
|
|
|
|
// Recover the race where attachment.* SSE arrives before the upload
|
|
// XHR's onload sets ``attachmentId``: walk recentEvents and watchdog
|
|
// the row so it can't stay stuck on 'processing'. Mirrors
|
|
// Upload.tsx's ``trackTraining``.
|
|
const trackAttachment = useCallback(
|
|
(clientId: string, attachmentId: string) => {
|
|
let handled = false;
|
|
|
|
const check = () => {
|
|
const state = store.getState();
|
|
const row = state.upload.attachments.find((a) => a.id === clientId);
|
|
if (!row) return true; // removed by user; stop tracking
|
|
if (row.status === 'completed' || row.status === 'failed') {
|
|
handled = true;
|
|
return true;
|
|
}
|
|
for (const event of state.notifications.recentEvents) {
|
|
if (event.scope?.id !== attachmentId) continue;
|
|
if (event.type === 'attachment.completed') {
|
|
const payload = (event.payload || {}) as Record<string, unknown>;
|
|
const tokenCount = Number(payload.token_count);
|
|
handled = true;
|
|
dispatch(
|
|
updateAttachment({
|
|
id: clientId,
|
|
updates: {
|
|
status: 'completed',
|
|
progress: 100,
|
|
...(Number.isFinite(tokenCount)
|
|
? { token_count: tokenCount }
|
|
: {}),
|
|
...(typeof payload.mime_type === 'string'
|
|
? { mimeType: payload.mime_type }
|
|
: {}),
|
|
...(typeof payload.extraction_status === 'string'
|
|
? { extractionStatus: payload.extraction_status }
|
|
: {}),
|
|
},
|
|
}),
|
|
);
|
|
return true;
|
|
}
|
|
if (event.type === 'attachment.failed') {
|
|
handled = true;
|
|
dispatch(
|
|
updateAttachment({
|
|
id: clientId,
|
|
updates: { status: 'failed' },
|
|
}),
|
|
);
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
};
|
|
|
|
if (check()) return;
|
|
const MAX_WAIT_MS = 5 * 60_000;
|
|
let unsubscribe: (() => void) | null = null;
|
|
const timer = window.setTimeout(() => {
|
|
unsubscribe?.();
|
|
if (!handled) {
|
|
handled = true;
|
|
console.warn(
|
|
'trackAttachment: timed out waiting for terminal SSE',
|
|
clientId,
|
|
attachmentId,
|
|
);
|
|
dispatch(
|
|
updateAttachment({
|
|
id: clientId,
|
|
updates: { status: 'failed' },
|
|
}),
|
|
);
|
|
}
|
|
}, MAX_WAIT_MS);
|
|
unsubscribe = store.subscribe(() => {
|
|
if (check()) {
|
|
window.clearTimeout(timer);
|
|
unsubscribe?.();
|
|
}
|
|
});
|
|
},
|
|
[dispatch, store],
|
|
);
|
|
|
|
const uploadFiles = useCallback(
|
|
async (incomingFiles: File[]) => {
|
|
if (!incomingFiles || incomingFiles.length === 0) return;
|
|
|
|
// Run the server's own rule here, not just the input's `accept`:
|
|
// mobile pickers ignore `accept`, and a file the server will refuse
|
|
// should say so before it costs an upload. Surface the refusal as a
|
|
// failed chip so the user sees why instead of a silent drop.
|
|
const { supported, unsupported } =
|
|
await partitionAttachmentFiles(incomingFiles);
|
|
unsupported.forEach((file) => {
|
|
dispatch(
|
|
addAttachment({
|
|
id: generateId(),
|
|
fileName: file.name,
|
|
progress: 0,
|
|
status: 'failed' as const,
|
|
taskId: '',
|
|
errorMessage: t('conversation.attachments.unsupportedType', {
|
|
extension: getFileExtension(file.name) || '?',
|
|
}),
|
|
}),
|
|
);
|
|
});
|
|
if (supported.length === 0) return;
|
|
const files = supported;
|
|
|
|
const apiHost = envVar('VITE_API_HOST');
|
|
|
|
if (files.length > 1) {
|
|
const formData = new FormData();
|
|
const indexToUiId: Record<number, string> = {};
|
|
|
|
files.forEach((file, i) => {
|
|
formData.append('file', file);
|
|
const uiId = generateId();
|
|
indexToUiId[i] = uiId;
|
|
dispatch(
|
|
addAttachment({
|
|
id: uiId,
|
|
fileName: file.name,
|
|
progress: 0,
|
|
status: 'uploading' as const,
|
|
taskId: '',
|
|
}),
|
|
);
|
|
});
|
|
|
|
const xhr = new XMLHttpRequest();
|
|
|
|
xhr.upload.addEventListener('progress', (event) => {
|
|
if (event.lengthComputable) {
|
|
const progress = Math.round((event.loaded / event.total) * 100);
|
|
Object.values(indexToUiId).forEach((uiId) =>
|
|
dispatch(
|
|
updateAttachment({
|
|
id: uiId,
|
|
updates: { progress },
|
|
}),
|
|
),
|
|
);
|
|
}
|
|
});
|
|
|
|
xhr.onload = () => {
|
|
const status = xhr.status;
|
|
if (status === 200) {
|
|
try {
|
|
const response = JSON.parse(xhr.responseText);
|
|
|
|
if (Array.isArray(response?.tasks)) {
|
|
const tasks = response.tasks as Array<{
|
|
task_id?: string;
|
|
filename?: string;
|
|
attachment_id?: string;
|
|
path?: string;
|
|
upload_index?: number;
|
|
}>;
|
|
const errors = Array.isArray(response?.errors)
|
|
? (response.errors as Array<{
|
|
filename?: string;
|
|
error?: string;
|
|
upload_index?: number;
|
|
}>)
|
|
: [];
|
|
const hasIndexedResults =
|
|
tasks.some((task) => typeof task.upload_index === 'number') ||
|
|
errors.some(
|
|
(errorItem) => typeof errorItem.upload_index === 'number',
|
|
);
|
|
|
|
if (hasIndexedResults) {
|
|
const tasksByIndex = new Map<
|
|
number,
|
|
(typeof tasks)[number]
|
|
>();
|
|
const failedIndices = new Set<number>();
|
|
|
|
tasks.forEach((task, taskOrderIndex) => {
|
|
const uploadIndex =
|
|
typeof task.upload_index === 'number'
|
|
? task.upload_index
|
|
: taskOrderIndex;
|
|
tasksByIndex.set(uploadIndex, task);
|
|
});
|
|
|
|
const errorsByIndex = new Map<number, string | undefined>();
|
|
errors.forEach((errorItem) => {
|
|
if (typeof errorItem.upload_index === 'number') {
|
|
failedIndices.add(errorItem.upload_index);
|
|
errorsByIndex.set(
|
|
errorItem.upload_index,
|
|
errorItem.error,
|
|
);
|
|
}
|
|
});
|
|
|
|
files.forEach((_, index) => {
|
|
const uiId = indexToUiId[index];
|
|
if (!uiId) return;
|
|
|
|
const task = tasksByIndex.get(index);
|
|
if (task?.task_id) {
|
|
dispatch(
|
|
updateAttachment({
|
|
id: uiId,
|
|
updates: {
|
|
taskId: task.task_id,
|
|
// Stash the server's attachment id so SSE
|
|
// ``attachment.*`` events can match this
|
|
// row by ``scope.id`` and drive the
|
|
// per-attachment push-fresh poll gate.
|
|
attachmentId: task.attachment_id,
|
|
status: 'processing',
|
|
progress: 10,
|
|
},
|
|
}),
|
|
);
|
|
if (task.attachment_id) {
|
|
trackAttachment(uiId, task.attachment_id);
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (failedIndices.has(index)) {
|
|
dispatch(
|
|
updateAttachment({
|
|
id: uiId,
|
|
updates: {
|
|
status: 'failed',
|
|
errorMessage: errorsByIndex.get(index),
|
|
},
|
|
}),
|
|
);
|
|
return;
|
|
}
|
|
|
|
dispatch(
|
|
updateAttachment({
|
|
id: uiId,
|
|
updates: { status: 'failed' },
|
|
}),
|
|
);
|
|
});
|
|
} else {
|
|
tasks.forEach((t, idx) => {
|
|
const uiId = indexToUiId[idx];
|
|
if (!uiId) return;
|
|
if (t?.task_id) {
|
|
dispatch(
|
|
updateAttachment({
|
|
id: uiId,
|
|
updates: {
|
|
taskId: t.task_id,
|
|
attachmentId: t.attachment_id,
|
|
status: 'processing',
|
|
progress: 10,
|
|
},
|
|
}),
|
|
);
|
|
if (t.attachment_id) {
|
|
trackAttachment(uiId, t.attachment_id);
|
|
}
|
|
} else {
|
|
dispatch(
|
|
updateAttachment({
|
|
id: uiId,
|
|
updates: { status: 'failed' },
|
|
}),
|
|
);
|
|
}
|
|
});
|
|
|
|
if (tasks.length < files.length) {
|
|
for (let i = tasks.length; i < files.length; i++) {
|
|
const uiId = indexToUiId[i];
|
|
if (uiId) {
|
|
dispatch(
|
|
updateAttachment({
|
|
id: uiId,
|
|
updates: { status: 'failed' },
|
|
}),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} else if (response?.task_id) {
|
|
if (files.length === 1) {
|
|
const uiId = indexToUiId[0];
|
|
if (uiId) {
|
|
dispatch(
|
|
updateAttachment({
|
|
id: uiId,
|
|
updates: {
|
|
taskId: response.task_id,
|
|
attachmentId: response.attachment_id,
|
|
status: 'processing',
|
|
progress: 10,
|
|
},
|
|
}),
|
|
);
|
|
if (response.attachment_id) {
|
|
trackAttachment(uiId, response.attachment_id);
|
|
}
|
|
}
|
|
} else {
|
|
console.warn(
|
|
'Server returned a single task_id for multiple files. Update backend to return tasks[].',
|
|
);
|
|
const firstUi = indexToUiId[0];
|
|
if (firstUi) {
|
|
dispatch(
|
|
updateAttachment({
|
|
id: firstUi,
|
|
updates: {
|
|
taskId: response.task_id,
|
|
status: 'processing',
|
|
progress: 10,
|
|
},
|
|
}),
|
|
);
|
|
}
|
|
for (let i = 1; i < files.length; i++) {
|
|
const uiId = indexToUiId[i];
|
|
if (uiId) {
|
|
dispatch(
|
|
updateAttachment({
|
|
id: uiId,
|
|
updates: { status: 'failed' },
|
|
}),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
console.error('Unexpected upload response shape', response);
|
|
Object.values(indexToUiId).forEach((id) =>
|
|
dispatch(
|
|
updateAttachment({
|
|
id,
|
|
updates: { status: 'failed' },
|
|
}),
|
|
),
|
|
);
|
|
}
|
|
} catch (err) {
|
|
console.error(
|
|
'Failed to parse upload response',
|
|
err,
|
|
xhr.responseText,
|
|
);
|
|
Object.values(indexToUiId).forEach((id) =>
|
|
dispatch(
|
|
updateAttachment({
|
|
id,
|
|
updates: { status: 'failed' },
|
|
}),
|
|
),
|
|
);
|
|
}
|
|
} else {
|
|
console.error('Upload failed', status, xhr.responseText);
|
|
// Each file gets its own reason where the server sent one; the
|
|
// top-level message is the fallback, not the answer for all of
|
|
// them — a batch can fail two files for two different reasons.
|
|
const fallbackMessage = parseUploadErrorMessage(xhr.responseText);
|
|
const errorsByIndex = parseUploadErrorsByIndex(xhr.responseText);
|
|
Object.entries(indexToUiId).forEach(([index, id]) =>
|
|
dispatch(
|
|
updateAttachment({
|
|
id,
|
|
updates: {
|
|
status: 'failed',
|
|
errorMessage:
|
|
errorsByIndex.get(Number(index)) ?? fallbackMessage,
|
|
},
|
|
}),
|
|
),
|
|
);
|
|
}
|
|
};
|
|
|
|
xhr.onerror = () => {
|
|
console.error('Upload network error');
|
|
Object.values(indexToUiId).forEach((id) =>
|
|
dispatch(
|
|
updateAttachment({
|
|
id,
|
|
updates: { status: 'failed' },
|
|
}),
|
|
),
|
|
);
|
|
};
|
|
|
|
xhr.open('POST', `${apiHost}${endpoints.USER.STORE_ATTACHMENT}`);
|
|
if (token) xhr.setRequestHeader('Authorization', `Bearer ${token}`);
|
|
xhr.send(formData);
|
|
return;
|
|
}
|
|
|
|
// Single-file path: upload each file individually (original repo behavior)
|
|
files.forEach((file) => {
|
|
const formData = new FormData();
|
|
formData.append('file', file);
|
|
const xhr = new XMLHttpRequest();
|
|
const uniqueId = generateId();
|
|
|
|
const newAttachment = {
|
|
id: uniqueId,
|
|
fileName: file.name,
|
|
progress: 0,
|
|
status: 'uploading' as const,
|
|
taskId: '',
|
|
};
|
|
|
|
dispatch(addAttachment(newAttachment));
|
|
|
|
xhr.upload.addEventListener('progress', (event) => {
|
|
if (event.lengthComputable) {
|
|
const progress = Math.round((event.loaded / event.total) * 100);
|
|
dispatch(
|
|
updateAttachment({
|
|
id: uniqueId,
|
|
updates: { progress },
|
|
}),
|
|
);
|
|
}
|
|
});
|
|
|
|
xhr.onload = () => {
|
|
if (xhr.status === 200) {
|
|
try {
|
|
const response = JSON.parse(xhr.responseText);
|
|
if (response.task_id) {
|
|
dispatch(
|
|
updateAttachment({
|
|
id: uniqueId,
|
|
updates: {
|
|
taskId: response.task_id,
|
|
attachmentId: response.attachment_id,
|
|
status: 'processing',
|
|
progress: 10,
|
|
},
|
|
}),
|
|
);
|
|
if (response.attachment_id) {
|
|
trackAttachment(uniqueId, response.attachment_id);
|
|
}
|
|
} else {
|
|
// If backend returned tasks[] for single-file, handle gracefully:
|
|
if (
|
|
Array.isArray(response?.tasks) &&
|
|
response.tasks[0]?.task_id
|
|
) {
|
|
dispatch(
|
|
updateAttachment({
|
|
id: uniqueId,
|
|
updates: {
|
|
taskId: response.tasks[0].task_id,
|
|
attachmentId: response.tasks[0].attachment_id,
|
|
status: 'processing',
|
|
progress: 10,
|
|
},
|
|
}),
|
|
);
|
|
if (response.tasks[0].attachment_id) {
|
|
trackAttachment(uniqueId, response.tasks[0].attachment_id);
|
|
}
|
|
} else {
|
|
dispatch(
|
|
updateAttachment({
|
|
id: uniqueId,
|
|
updates: { status: 'failed' },
|
|
}),
|
|
);
|
|
}
|
|
}
|
|
} catch (err) {
|
|
console.error(
|
|
'Failed to parse upload response',
|
|
err,
|
|
xhr.responseText,
|
|
);
|
|
dispatch(
|
|
updateAttachment({
|
|
id: uniqueId,
|
|
updates: { status: 'failed' },
|
|
}),
|
|
);
|
|
}
|
|
} else {
|
|
dispatch(
|
|
updateAttachment({
|
|
id: uniqueId,
|
|
updates: {
|
|
status: 'failed',
|
|
errorMessage: parseUploadErrorMessage(xhr.responseText),
|
|
},
|
|
}),
|
|
);
|
|
}
|
|
};
|
|
|
|
xhr.onerror = () => {
|
|
dispatch(
|
|
updateAttachment({
|
|
id: uniqueId,
|
|
updates: { status: 'failed' },
|
|
}),
|
|
);
|
|
};
|
|
|
|
xhr.open('POST', `${apiHost}${endpoints.USER.STORE_ATTACHMENT}`);
|
|
if (token) xhr.setRequestHeader('Authorization', `Bearer ${token}`);
|
|
xhr.send(formData);
|
|
});
|
|
},
|
|
[dispatch, t, token, trackAttachment],
|
|
);
|
|
|
|
const handleFileAttachment = (e: React.ChangeEvent<HTMLInputElement>) => {
|
|
if (!e.target.files || e.target.files.length === 0) return;
|
|
const files = Array.from(e.target.files);
|
|
uploadFiles(files);
|
|
// clear input so same file can be selected again
|
|
e.target.value = '';
|
|
};
|
|
|
|
// Drag & drop via react-dropzone
|
|
const onDrop = useCallback(
|
|
(acceptedFiles: File[]) => {
|
|
uploadFiles(acceptedFiles);
|
|
setHandleDragActive(false);
|
|
},
|
|
[uploadFiles],
|
|
);
|
|
|
|
const { getRootProps, getInputProps } = useDropzone({
|
|
onDrop,
|
|
noClick: true,
|
|
noKeyboard: true,
|
|
// The textarea below owns paste-to-attach via handlePaste. react-dropzone
|
|
// added its own paste handling in v19.2 (on by default), which fires on the
|
|
// root for pastes into any focused descendant - so both would upload the
|
|
// same file. Leave paste to handlePaste.
|
|
noPaste: true,
|
|
multiple: true,
|
|
onDragEnter: () => {
|
|
setHandleDragActive(true);
|
|
},
|
|
onDragLeave: () => {
|
|
setHandleDragActive(false);
|
|
},
|
|
maxSize: 25000000,
|
|
// No `accept`: react-dropzone would drop a rejected file on the floor
|
|
// with no feedback, and its mime matching disagrees with the server for
|
|
// text files that have no parser (.py, .log). uploadFiles applies the
|
|
// server's rule and reports what it refuses.
|
|
});
|
|
|
|
const handleInput = useCallback(() => {
|
|
if (!inputRef.current) return;
|
|
if (window.innerWidth < 350) inputRef.current.style.height = 'auto';
|
|
else inputRef.current.style.height = '64px';
|
|
inputRef.current.style.height = `${Math.min(
|
|
inputRef.current.scrollHeight,
|
|
Math.round(window.innerHeight * 0.4),
|
|
)}px`;
|
|
}, []);
|
|
|
|
const buildVoiceDraftValue = (baseText: string, transcript: string) => {
|
|
const normalizedBaseText = baseText ?? '';
|
|
const normalizedTranscript = transcript.trim();
|
|
|
|
if (!normalizedTranscript) {
|
|
return normalizedBaseText;
|
|
}
|
|
|
|
return normalizedBaseText.trim()
|
|
? `${normalizedBaseText}${
|
|
normalizedBaseText.endsWith('\n') ? '' : '\n'
|
|
}${normalizedTranscript}`
|
|
: normalizedTranscript;
|
|
};
|
|
|
|
const applyLiveTranscript = (transcript: string) => {
|
|
const normalizedTranscript = transcript.trim();
|
|
liveTranscriptRef.current = normalizedTranscript;
|
|
setValue(
|
|
buildVoiceDraftValue(voiceBaseValueRef.current, normalizedTranscript),
|
|
);
|
|
};
|
|
|
|
const promptVoiceFileFallback = (message: string) => {
|
|
setRecordingState('idle');
|
|
setVoiceError(`${message} Choose or record an audio file instead.`);
|
|
setTimeout(() => {
|
|
voiceFileInputRef.current?.click();
|
|
}, 0);
|
|
};
|
|
|
|
const transcribeUploadedAudioFile = async (file: File) => {
|
|
try {
|
|
setVoiceError(null);
|
|
setRecordingState('transcribing');
|
|
voiceBaseValueRef.current = value;
|
|
liveTranscriptRef.current = '';
|
|
|
|
const response = await userService.transcribeAudio(file, token);
|
|
const data = await response.json();
|
|
|
|
if (!response.ok || !data?.success) {
|
|
throw new Error(data?.message || 'Failed to transcribe audio.');
|
|
}
|
|
|
|
if (typeof data.text !== 'string' || !data.text.trim()) {
|
|
throw new Error('No transcript was returned for this audio file.');
|
|
}
|
|
|
|
applyLiveTranscript(data.text);
|
|
setRecordingState('idle');
|
|
if (autoFocus) {
|
|
setTimeout(() => {
|
|
inputRef.current?.focus();
|
|
}, 0);
|
|
}
|
|
} catch (error) {
|
|
console.error('Uploaded audio transcription failed', error);
|
|
setRecordingState('error');
|
|
setVoiceError(
|
|
error instanceof Error ? error.message : 'Failed to transcribe audio.',
|
|
);
|
|
}
|
|
};
|
|
|
|
const trimLivePcmBuffer = () => {
|
|
const maxBufferedSamples =
|
|
LIVE_CAPTURE_SAMPLE_RATE * LIVE_CAPTURE_MAX_BUFFER_SECONDS;
|
|
|
|
while (
|
|
totalBufferedSamplesRef.current > maxBufferedSamples &&
|
|
pcmChunksRef.current.length > 1
|
|
) {
|
|
const removedChunk = pcmChunksRef.current.shift();
|
|
if (!removedChunk) {
|
|
break;
|
|
}
|
|
totalBufferedSamplesRef.current -= removedChunk.length;
|
|
}
|
|
|
|
if (
|
|
totalBufferedSamplesRef.current > maxBufferedSamples &&
|
|
pcmChunksRef.current.length === 1
|
|
) {
|
|
const onlyChunk = pcmChunksRef.current[0];
|
|
if (!onlyChunk || onlyChunk.length <= maxBufferedSamples) {
|
|
return;
|
|
}
|
|
|
|
const trimmedChunk = onlyChunk.slice(
|
|
onlyChunk.length - maxBufferedSamples,
|
|
);
|
|
pcmChunksRef.current = [trimmedChunk];
|
|
totalBufferedSamplesRef.current = trimmedChunk.length;
|
|
}
|
|
};
|
|
|
|
const cleanupLiveSession = async () => {
|
|
const sessionId = liveSessionIdRef.current;
|
|
if (!sessionId) {
|
|
return;
|
|
}
|
|
|
|
liveSessionIdRef.current = null;
|
|
try {
|
|
await userService.finishLiveTranscription(sessionId, token);
|
|
} catch {
|
|
// Best-effort cleanup only.
|
|
}
|
|
};
|
|
|
|
const failLiveTranscription = async (message: string) => {
|
|
console.error('Live audio transcription failed', message);
|
|
stopAudioProcessing();
|
|
await cleanupLiveSession();
|
|
resetLiveTranscriptionState();
|
|
setRecordingState('error');
|
|
setVoiceError(message);
|
|
};
|
|
|
|
const finalizeLiveTranscription = async () => {
|
|
const sessionId = liveSessionIdRef.current;
|
|
if (!sessionId) {
|
|
resetLiveTranscriptionState();
|
|
setRecordingState('idle');
|
|
return;
|
|
}
|
|
|
|
try {
|
|
const response = await userService.finishLiveTranscription(
|
|
sessionId,
|
|
token,
|
|
);
|
|
const data = await response.json();
|
|
|
|
if (!response.ok || !data?.success) {
|
|
throw new Error(
|
|
data?.message || 'Failed to finalize live transcription.',
|
|
);
|
|
}
|
|
|
|
if (typeof data.text === 'string') {
|
|
applyLiveTranscript(data.text);
|
|
}
|
|
|
|
setRecordingState('idle');
|
|
if (autoFocus) {
|
|
setTimeout(() => {
|
|
inputRef.current?.focus();
|
|
}, 0);
|
|
}
|
|
} catch (error) {
|
|
console.error('Finalizing live audio transcription failed', error);
|
|
setRecordingState('error');
|
|
setVoiceError(
|
|
error instanceof Error
|
|
? error.message
|
|
: 'Failed to finalize live transcription.',
|
|
);
|
|
} finally {
|
|
resetLiveTranscriptionState();
|
|
}
|
|
};
|
|
|
|
const maybeFinalizeLiveTranscription = async () => {
|
|
if (
|
|
!liveStopRequestedRef.current ||
|
|
liveUploadInFlightRef.current ||
|
|
livePendingSnapshotRef.current
|
|
) {
|
|
return;
|
|
}
|
|
|
|
await finalizeLiveTranscription();
|
|
};
|
|
|
|
const processPendingLiveSnapshot = async () => {
|
|
if (liveUploadInFlightRef.current) {
|
|
return;
|
|
}
|
|
|
|
const nextSnapshot = livePendingSnapshotRef.current;
|
|
const sessionId = liveSessionIdRef.current;
|
|
if (!nextSnapshot || !sessionId) {
|
|
await maybeFinalizeLiveTranscription();
|
|
return;
|
|
}
|
|
|
|
livePendingSnapshotRef.current = null;
|
|
liveUploadInFlightRef.current = true;
|
|
|
|
try {
|
|
const file = new File(
|
|
[nextSnapshot.blob],
|
|
`voice-live-${nextSnapshot.chunkIndex}.wav`,
|
|
{
|
|
type: 'audio/wav',
|
|
},
|
|
);
|
|
const response = await userService.transcribeLiveAudioChunk(
|
|
sessionId,
|
|
nextSnapshot.chunkIndex,
|
|
file,
|
|
token,
|
|
nextSnapshot.isSilence,
|
|
);
|
|
const data = await response.json();
|
|
|
|
if (!response.ok || !data?.success) {
|
|
throw new Error(data?.message || 'Failed to transcribe audio.');
|
|
}
|
|
|
|
if (typeof data.transcript_text === 'string') {
|
|
applyLiveTranscript(data.transcript_text);
|
|
}
|
|
} catch (error) {
|
|
await failLiveTranscription(
|
|
error instanceof Error ? error.message : 'Failed to transcribe audio.',
|
|
);
|
|
return;
|
|
} finally {
|
|
liveUploadInFlightRef.current = false;
|
|
}
|
|
|
|
if (livePendingSnapshotRef.current) {
|
|
void processPendingLiveSnapshot();
|
|
return;
|
|
}
|
|
|
|
void maybeFinalizeLiveTranscription();
|
|
};
|
|
|
|
const queueCurrentLiveSnapshot = (forceSilence = false) => {
|
|
if (
|
|
totalCapturedSamplesRef.current === lastSnapshotCapturedSamplesRef.current
|
|
) {
|
|
return;
|
|
}
|
|
|
|
if (!pcmChunksRef.current.length || totalBufferedSamplesRef.current <= 0) {
|
|
return;
|
|
}
|
|
|
|
const pcmSnapshot = concatenateFloat32Chunks(
|
|
pcmChunksRef.current,
|
|
totalBufferedSamplesRef.current,
|
|
);
|
|
if (!pcmSnapshot.length) {
|
|
return;
|
|
}
|
|
|
|
const { sumSquares, sampleCount } = recentWindowRmsRef.current;
|
|
const averageRms =
|
|
sampleCount > 0 ? Math.sqrt(sumSquares / sampleCount) : 0;
|
|
const isSilence = forceSilence || averageRms < LIVE_SILENCE_RMS_THRESHOLD;
|
|
|
|
recentWindowRmsRef.current = { sumSquares: 0, sampleCount: 0 };
|
|
lastSnapshotCapturedSamplesRef.current = totalCapturedSamplesRef.current;
|
|
livePendingSnapshotRef.current = {
|
|
blob: encodeWavFromFloat32(pcmSnapshot, LIVE_CAPTURE_SAMPLE_RATE),
|
|
chunkIndex: liveChunkIndexRef.current,
|
|
isSilence,
|
|
};
|
|
liveChunkIndexRef.current += 1;
|
|
void processPendingLiveSnapshot();
|
|
};
|
|
|
|
const handleVoiceInput = async () => {
|
|
if (recordingState === 'transcribing') {
|
|
return;
|
|
}
|
|
|
|
if (recordingState === 'recording') {
|
|
setRecordingState('transcribing');
|
|
liveStopRequestedRef.current = true;
|
|
stopAudioProcessing();
|
|
queueCurrentLiveSnapshot();
|
|
void maybeFinalizeLiveTranscription();
|
|
return;
|
|
}
|
|
|
|
const voiceInputSupportError = getVoiceInputSupportError();
|
|
if (voiceInputSupportError) {
|
|
promptVoiceFileFallback(voiceInputSupportError);
|
|
return;
|
|
}
|
|
|
|
const AudioContextConstructor = getAudioContextConstructor();
|
|
if (!AudioContextConstructor) {
|
|
setRecordingState('error');
|
|
setVoiceError('Voice input requires Web Audio support in this browser.');
|
|
return;
|
|
}
|
|
|
|
let stream: MediaStream | null = null;
|
|
try {
|
|
setVoiceError(null);
|
|
stream = await getUserMediaStream({ audio: true });
|
|
} catch (error) {
|
|
promptVoiceFileFallback(getVoiceInputErrorMessage(error));
|
|
return;
|
|
}
|
|
|
|
try {
|
|
const liveStartResponse = await userService.startLiveTranscription(token);
|
|
const liveStartData = await liveStartResponse.json();
|
|
if (!liveStartResponse.ok || !liveStartData?.success) {
|
|
throw new Error(
|
|
liveStartData?.message || 'Failed to start live transcription.',
|
|
);
|
|
}
|
|
|
|
const audioContext = new AudioContextConstructor();
|
|
await audioContext.resume().catch(() => undefined);
|
|
const sourceNode = audioContext.createMediaStreamSource(stream);
|
|
const processorNode = audioContext.createScriptProcessor(4096, 1, 1);
|
|
const silenceGain = audioContext.createGain();
|
|
silenceGain.gain.value = 0;
|
|
|
|
pcmChunksRef.current = [];
|
|
totalBufferedSamplesRef.current = 0;
|
|
totalCapturedSamplesRef.current = 0;
|
|
lastSnapshotCapturedSamplesRef.current = 0;
|
|
recentWindowRmsRef.current = { sumSquares: 0, sampleCount: 0 };
|
|
liveSessionIdRef.current = liveStartData.session_id;
|
|
livePendingSnapshotRef.current = null;
|
|
liveChunkIndexRef.current = 0;
|
|
liveUploadInFlightRef.current = false;
|
|
liveStopRequestedRef.current = false;
|
|
voiceBaseValueRef.current = value;
|
|
liveTranscriptRef.current = '';
|
|
applyLiveTranscript('');
|
|
|
|
processorNode.onaudioprocess = (event: AudioProcessingEvent) => {
|
|
const inputData = event.inputBuffer.getChannelData(0);
|
|
if (!inputData.length) {
|
|
return;
|
|
}
|
|
|
|
const capturedChunk = new Float32Array(inputData.length);
|
|
capturedChunk.set(inputData);
|
|
|
|
const downsampledChunk = downsampleFloat32Buffer(
|
|
capturedChunk,
|
|
audioContext.sampleRate,
|
|
LIVE_CAPTURE_SAMPLE_RATE,
|
|
);
|
|
if (!downsampledChunk.length) {
|
|
return;
|
|
}
|
|
|
|
pcmChunksRef.current.push(downsampledChunk);
|
|
totalBufferedSamplesRef.current += downsampledChunk.length;
|
|
totalCapturedSamplesRef.current += downsampledChunk.length;
|
|
|
|
let sumSquares = 0;
|
|
for (let index = 0; index < downsampledChunk.length; index += 1) {
|
|
const sample = downsampledChunk[index];
|
|
sumSquares += sample * sample;
|
|
}
|
|
|
|
recentWindowRmsRef.current.sumSquares += sumSquares;
|
|
recentWindowRmsRef.current.sampleCount += downsampledChunk.length;
|
|
trimLivePcmBuffer();
|
|
};
|
|
|
|
sourceNode.connect(processorNode);
|
|
processorNode.connect(silenceGain);
|
|
silenceGain.connect(audioContext.destination);
|
|
|
|
mediaStreamRef.current = stream;
|
|
audioContextRef.current = audioContext;
|
|
audioSourceNodeRef.current = sourceNode;
|
|
audioProcessorNodeRef.current = processorNode;
|
|
audioSilenceGainRef.current = silenceGain;
|
|
snapshotIntervalRef.current = window.setInterval(() => {
|
|
if (!liveStopRequestedRef.current) {
|
|
queueCurrentLiveSnapshot();
|
|
}
|
|
}, LIVE_TRANSCRIPTION_TIMESLICE_MS);
|
|
|
|
setRecordingState('recording');
|
|
} catch (error) {
|
|
console.error('Live voice transcription failed', error);
|
|
stream?.getTracks().forEach((track) => track.stop());
|
|
stopAudioProcessing();
|
|
await cleanupLiveSession();
|
|
resetLiveTranscriptionState();
|
|
setRecordingState('error');
|
|
setVoiceError(
|
|
error instanceof Error
|
|
? error.message
|
|
: 'Failed to start live transcription.',
|
|
);
|
|
}
|
|
};
|
|
|
|
const isMountedRef = useRef(true);
|
|
|
|
useEffect(() => {
|
|
isMountedRef.current = true;
|
|
return () => {
|
|
isMountedRef.current = false;
|
|
};
|
|
}, []);
|
|
|
|
useLayoutEffect(() => {
|
|
handleInput();
|
|
}, [value, handleInput]);
|
|
|
|
useEffect(() => {
|
|
window.addEventListener('resize', handleInput);
|
|
return () => window.removeEventListener('resize', handleInput);
|
|
}, [handleInput]);
|
|
|
|
const handleChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => {
|
|
setValue(e.target.value);
|
|
};
|
|
|
|
const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
|
|
if (e.key === 'Enter' && !e.shiftKey) {
|
|
e.preventDefault();
|
|
handleSubmit();
|
|
}
|
|
};
|
|
|
|
const handlePaste = (e: React.ClipboardEvent<HTMLTextAreaElement>) => {
|
|
const clipboardItems = e.clipboardData?.items;
|
|
const files: File[] = [];
|
|
|
|
if (!clipboardItems) return;
|
|
|
|
for (let i = 0; i < clipboardItems.length; i++) {
|
|
const item = clipboardItems[i];
|
|
|
|
if (item.kind === 'file') {
|
|
const file = item.getAsFile();
|
|
if (file) {
|
|
files.push(file);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (files.length > 0) {
|
|
// Prevent weird binary stuff from being pasted as text
|
|
e.preventDefault();
|
|
uploadFiles(files);
|
|
}
|
|
};
|
|
|
|
const handleVoiceFileAttachment = (
|
|
e: React.ChangeEvent<HTMLInputElement>,
|
|
) => {
|
|
const file = e.target.files?.[0];
|
|
e.target.value = '';
|
|
|
|
if (!file) {
|
|
return;
|
|
}
|
|
|
|
void transcribeUploadedAudioFile(file);
|
|
};
|
|
|
|
const handlePostDocumentSelect = (_docs: Doc[] | null) => {
|
|
// Hook point for downstream side-effects after a source is toggled.
|
|
void _docs;
|
|
};
|
|
|
|
const sourceItems: MultiSelectPopoverItem[] = useMemo(
|
|
() =>
|
|
toSourcePickerItems(
|
|
sourceDocs,
|
|
{
|
|
own: t('conversation.sources.groupOwn'),
|
|
team: t('conversation.sources.groupTeam'),
|
|
},
|
|
SourceIcon,
|
|
),
|
|
[sourceDocs, t],
|
|
);
|
|
|
|
const selectedSourceIds = (
|
|
selectedDocs && Array.isArray(selectedDocs) ? selectedDocs : []
|
|
).map((doc) => sourceItemId(doc));
|
|
|
|
const handleToggleSource = (id: string) => {
|
|
if (!sourceDocs) return;
|
|
const current = Array.isArray(selectedDocs) ? selectedDocs : [];
|
|
const matched = current.find((doc) => sourceItemId(doc) === id);
|
|
let updated: Doc[];
|
|
if (matched) {
|
|
updated = current.filter((doc) => sourceItemId(doc) !== id);
|
|
} else {
|
|
const incoming = sourceDocs.find((doc) => sourceItemId(doc) === id);
|
|
if (!incoming) return;
|
|
updated = [...current, incoming];
|
|
}
|
|
dispatch(setSelectedDocs(updated.length > 0 ? updated : []));
|
|
handlePostDocumentSelect(updated.length > 0 ? updated : null);
|
|
};
|
|
|
|
const fetchUserTools = useCallback(() => {
|
|
setToolsLoading(true);
|
|
userService
|
|
.getUserTools(token)
|
|
.then((res) => res.json())
|
|
.then((data) => {
|
|
const filtered = (data.tools || []).filter(isChatToolVisible);
|
|
setUserTools(filtered);
|
|
})
|
|
.catch((error) => {
|
|
console.error('Error fetching tools:', error);
|
|
})
|
|
.finally(() => setToolsLoading(false));
|
|
}, [token]);
|
|
|
|
useEffect(() => {
|
|
if (isToolsPopupOpen) fetchUserTools();
|
|
}, [isToolsPopupOpen, fetchUserTools]);
|
|
|
|
const toolItems: MultiSelectPopoverItem[] = userTools.map((tool) => ({
|
|
id: tool.id,
|
|
label: tool.customName || tool.displayName,
|
|
icon: <ToolIcon name={tool.name} className="h-5 w-5" />,
|
|
}));
|
|
|
|
const selectedToolIds = userTools
|
|
.filter((tool) => tool.status)
|
|
.map((tool) => tool.id);
|
|
|
|
const handleToggleTool = (id: string) => {
|
|
const tool = userTools.find((t) => t.id === id);
|
|
if (!tool) return;
|
|
const newStatus = !tool.status;
|
|
userService
|
|
.updateToolStatus({ id, status: newStatus }, token)
|
|
.then(() => {
|
|
setUserTools((prev) =>
|
|
prev.map((t) => (t.id === id ? { ...t, status: newStatus } : t)),
|
|
);
|
|
})
|
|
.catch((error) => {
|
|
console.error('Failed to update tool status:', error);
|
|
});
|
|
};
|
|
|
|
const handleUploadClick = () => {
|
|
setUploadModalState('ACTIVE');
|
|
setIsSourcesPopupOpen(false);
|
|
};
|
|
|
|
// When ``allowSendWithoutText`` is set, an attachment-only submit is
|
|
// permitted as long as at least one attachment exists; a still-pending
|
|
// one arms the send instead of submitting (see handleSubmit).
|
|
const hasSubmittableContent =
|
|
Boolean(value.trim()) || (allowSendWithoutText && attachments.length > 0);
|
|
const canSubmit =
|
|
hasSubmittableContent &&
|
|
!loading &&
|
|
recordingState !== 'recording' &&
|
|
recordingState !== 'transcribing';
|
|
|
|
const submitNow = () => {
|
|
onSubmit(value);
|
|
setValue('');
|
|
if (isTouch) {
|
|
inputRef.current?.blur();
|
|
} else if (autoFocus) {
|
|
setTimeout(() => {
|
|
if (isMountedRef.current) {
|
|
inputRef.current?.focus();
|
|
}
|
|
}, 0);
|
|
}
|
|
};
|
|
|
|
const {
|
|
armed: sendArmed,
|
|
readiness: sendReadiness,
|
|
arm: armSend,
|
|
cancel: cancelArmedSend,
|
|
} = useArmedSend({ attachments, onFlush: submitNow });
|
|
|
|
// Adopt a question queued outside the composer: seed the input, arm,
|
|
// and hand the wait to the standard banner. If the attachments already
|
|
// resolved by the time this runs, the armed-send effect flushes at once.
|
|
useEffect(() => {
|
|
if (queuedQuestion == null || queuedQuestion === '') return;
|
|
setValue(queuedQuestion);
|
|
armSend();
|
|
onQueuedQuestionConsumed?.();
|
|
}, [queuedQuestion]);
|
|
|
|
const handleSubmit = () => {
|
|
if (!canSubmit) return;
|
|
// Attachments still uploading/parsing (or failed) must never be
|
|
// silently dropped from the payload: hold the send in the composer
|
|
// until every attachment resolves, then flush automatically.
|
|
if (sendReadiness.state !== 'ready') {
|
|
armSend();
|
|
return;
|
|
}
|
|
submitNow();
|
|
};
|
|
|
|
const handleCancel = () => {
|
|
handleAbort();
|
|
};
|
|
|
|
const [draggingId, setDraggingId] = useState<string | null>(null);
|
|
|
|
const findIndexById = (id: string) =>
|
|
attachments.findIndex((a) => a.id === id);
|
|
|
|
const handleDragStart = (e: React.DragEvent, id: string) => {
|
|
setDraggingId(id);
|
|
try {
|
|
e.dataTransfer.setData('text/plain', id);
|
|
e.dataTransfer.effectAllowed = 'move';
|
|
} catch {
|
|
// ignore
|
|
}
|
|
};
|
|
|
|
const handleDragOver = (e: React.DragEvent) => {
|
|
e.preventDefault();
|
|
e.dataTransfer.dropEffect = 'move';
|
|
};
|
|
|
|
const handleDropOn = (e: React.DragEvent, targetId: string) => {
|
|
e.preventDefault();
|
|
const sourceId = e.dataTransfer.getData('text/plain');
|
|
if (!sourceId || sourceId === targetId) return;
|
|
|
|
const sourceIndex = findIndexById(sourceId);
|
|
const destIndex = findIndexById(targetId);
|
|
if (sourceIndex === -1 || destIndex === -1) return;
|
|
|
|
dispatch(reorderAttachments({ sourceIndex, destinationIndex: destIndex }));
|
|
setDraggingId(null);
|
|
};
|
|
|
|
return (
|
|
<div {...getRootProps()} className="flex w-full flex-col">
|
|
{/* react-dropzone input (for drag/drop) */}
|
|
<input {...getInputProps()} />
|
|
<input
|
|
ref={voiceFileInputRef}
|
|
type="file"
|
|
className="hidden"
|
|
accept={AUDIO_FILE_ACCEPT_ATTR}
|
|
capture="user"
|
|
onChange={handleVoiceFileAttachment}
|
|
/>
|
|
|
|
{/* translate="no": keep Chrome's page translator out of the composer —
|
|
it rewrites text nodes into <font> wrappers and React loses the
|
|
controls (dead Attach button, see AttachFileButton). */}
|
|
<div
|
|
translate="no"
|
|
className="border-border bg-card relative flex w-full flex-col rounded-3xl border dark:bg-transparent"
|
|
>
|
|
<AttachmentChipList
|
|
attachments={attachments}
|
|
draggingId={draggingId}
|
|
onRemove={(id) => dispatch(removeAttachment(id))}
|
|
onDragStart={handleDragStart}
|
|
onDragOver={handleDragOver}
|
|
onDropOn={handleDropOn}
|
|
unreadableIds={unreadableAttachmentIds}
|
|
modelName={selectedModel?.display_name}
|
|
/>
|
|
|
|
{sendArmed && sendReadiness.state === 'waiting' && (
|
|
<div
|
|
className="text-muted-foreground flex items-center gap-2 px-2 pb-1 text-xs sm:px-3"
|
|
role="status"
|
|
>
|
|
<span>
|
|
{t('conversation.attachments.waitingToSend', {
|
|
count: sendReadiness.pendingCount,
|
|
})}
|
|
</span>
|
|
<button
|
|
type="button"
|
|
onClick={cancelArmedSend}
|
|
className="underline hover:opacity-80"
|
|
>
|
|
{t('conversation.attachments.cancelQueuedSend')}
|
|
</button>
|
|
</div>
|
|
)}
|
|
{sendArmed && sendReadiness.state === 'blocked' && (
|
|
<div
|
|
className="px-2 pb-1 text-xs text-[#B42318] sm:px-3"
|
|
role="alert"
|
|
>
|
|
{t('conversation.attachments.sendBlockedByFailed', {
|
|
names: sendReadiness.failedNames.join(', '),
|
|
})}
|
|
</div>
|
|
)}
|
|
{voiceError && (
|
|
<div className="px-2 pb-1 text-xs text-[#B42318] sm:px-3">
|
|
{voiceError}
|
|
</div>
|
|
)}
|
|
|
|
<div className="w-full">
|
|
<label htmlFor="message-input" className="sr-only">
|
|
{t('inputPlaceholder')}
|
|
</label>
|
|
<textarea
|
|
id="message-input"
|
|
ref={inputRef}
|
|
value={value}
|
|
autoFocus={autoFocus && !isTouch}
|
|
onChange={handleChange}
|
|
readOnly={
|
|
recordingState === 'recording' ||
|
|
recordingState === 'transcribing'
|
|
}
|
|
tabIndex={1}
|
|
placeholder={t('inputPlaceholder')}
|
|
className="inputbox-style dark:text-foreground dark:placeholder:text-muted-foreground/50 w-full scrollbar-thin overflow-x-hidden overflow-y-auto rounded-t-3xl bg-transparent px-2 text-base leading-tight whitespace-pre-wrap opacity-100 placeholder:text-gray-500 focus:outline-hidden sm:px-3"
|
|
onKeyDown={handleKeyDown}
|
|
onPaste={handlePaste}
|
|
aria-label={t('inputPlaceholder')}
|
|
/>
|
|
</div>
|
|
|
|
<div className="flex items-center px-2 pb-1.5 sm:px-3 sm:pb-2">
|
|
<div className="flex grow flex-wrap gap-1 sm:gap-2">
|
|
{showSourceButton && (
|
|
<SourcesTrigger
|
|
open={isSourcesPopupOpen}
|
|
onOpenChange={setIsSourcesPopupOpen}
|
|
items={sourceItems}
|
|
selectedIds={selectedSourceIds}
|
|
onToggle={handleToggleSource}
|
|
selectedDocs={selectedDocs}
|
|
onUploadClick={handleUploadClick}
|
|
/>
|
|
)}
|
|
|
|
{showToolButton && (
|
|
<ToolsTrigger
|
|
open={isToolsPopupOpen}
|
|
onOpenChange={setIsToolsPopupOpen}
|
|
items={toolItems}
|
|
selectedIds={selectedToolIds}
|
|
onToggle={handleToggleTool}
|
|
loading={toolsLoading}
|
|
/>
|
|
)}
|
|
{ENABLE_VOICE_INPUT && sttAvailable && (
|
|
<MicButton
|
|
recordingState={recordingState}
|
|
loading={loading}
|
|
onClick={() => {
|
|
void handleVoiceInput();
|
|
}}
|
|
/>
|
|
)}
|
|
<AttachFileButton onChange={handleFileAttachment} />
|
|
{/* Additional badges can be added here in the future */}
|
|
</div>
|
|
|
|
{loading ? (
|
|
<Button
|
|
type="button"
|
|
variant="default"
|
|
size="icon"
|
|
onClick={handleCancel}
|
|
aria-label={t('cancel')}
|
|
className="bg-primary ml-auto h-7 w-7 shrink-0 rounded-full text-white sm:h-9 sm:w-9"
|
|
disabled={!loading}
|
|
>
|
|
<div className="flex h-3 w-3 items-center justify-center rounded-sm bg-white sm:h-3.5 sm:w-3.5" />
|
|
</Button>
|
|
) : (
|
|
<Button
|
|
type="button"
|
|
variant="default"
|
|
size="icon"
|
|
onClick={handleSubmit}
|
|
aria-label={t('send')}
|
|
className={`ml-auto h-7 w-7 shrink-0 rounded-full transition-colors duration-300 ease-in-out sm:h-9 sm:w-9 ${
|
|
canSubmit
|
|
? 'bg-primary text-white'
|
|
: 'bg-muted text-muted-foreground dark:bg-accent dark:text-muted-foreground'
|
|
}`}
|
|
disabled={!canSubmit}
|
|
>
|
|
<SendArrow
|
|
className="mx-auto my-auto block h-3.5 w-3.5 sm:h-4 sm:w-4"
|
|
aria-label={t('send')}
|
|
role="img"
|
|
/>
|
|
</Button>
|
|
)}
|
|
</div>
|
|
</div>
|
|
|
|
{uploadModalState === 'ACTIVE' && (
|
|
<Upload
|
|
receivedFile={[]}
|
|
setModalState={setUploadModalState}
|
|
isOnboarding={false}
|
|
renderTab={null}
|
|
close={() => setUploadModalState('INACTIVE')}
|
|
/>
|
|
)}
|
|
|
|
{handleDragActive &&
|
|
createPortal(
|
|
<div className="dark:bg-background/85 pointer-events-none fixed top-0 left-0 z-50 flex size-full flex-col items-center justify-center bg-white/85">
|
|
<img className="filter dark:invert" src={DragFileUpload} />
|
|
<span className="text-muted-foreground dark:text-muted-foreground px-2 text-2xl font-bold">
|
|
{t('modals.uploadDoc.drag.title')}
|
|
</span>
|
|
<span className="text-s text-muted-foreground dark:text-muted-foreground w-48 p-2 text-center">
|
|
{t('modals.uploadDoc.drag.description')}
|
|
</span>
|
|
</div>,
|
|
document.body,
|
|
)}
|
|
</div>
|
|
);
|
|
}
|