From 7df153a79872338e618580159eb8a5edd2d2d44d Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Wed, 9 Sep 2026 05:13:57 +0530 Subject: [PATCH] feat(widget): attachments and voice input in the composer --- docs/content/Extensions/chat-widget.mdx | 32 +- extensions/react-widget/README.md | 12 +- extensions/react-widget/src/App.tsx | 5 +- .../src/components/ComposerControls.tsx | 623 ++++++++++++++++++ .../src/components/DocsGPTWidget.tsx | 463 ++++++++++--- .../react-widget/src/components/SearchBar.tsx | 8 +- .../src/components/defaultAvatar.ts | 12 + .../react-widget/src/components/tokens.ts | 8 + .../react-widget/src/hooks/useAttachments.ts | 224 +++++++ .../react-widget/src/hooks/useVoiceInput.ts | 245 +++++++ .../src/requests/attachmentsApi.ts | 128 ++++ .../react-widget/src/requests/streamingApi.ts | 6 +- extensions/react-widget/src/types/index.ts | 41 ++ 13 files changed, 1713 insertions(+), 94 deletions(-) create mode 100644 extensions/react-widget/src/components/ComposerControls.tsx create mode 100644 extensions/react-widget/src/components/defaultAvatar.ts create mode 100644 extensions/react-widget/src/components/tokens.ts create mode 100644 extensions/react-widget/src/hooks/useAttachments.ts create mode 100644 extensions/react-widget/src/hooks/useVoiceInput.ts create mode 100644 extensions/react-widget/src/requests/attachmentsApi.ts diff --git a/docs/content/Extensions/chat-widget.mdx b/docs/content/Extensions/chat-widget.mdx index 5fdf2157..200a54a8 100644 --- a/docs/content/Extensions/chat-widget.mdx +++ b/docs/content/Extensions/chat-widget.mdx @@ -47,7 +47,7 @@ Now, you can embed the widget within your React component's JSX: ` block, use the `renderDocsGPTWidget` functio renderDocsGPTWidget('app', { apiHost: 'https://gptcloud.arc53.com', // self-hosted: your API URL, e.g. http://localhost:7091 apiKey: "your-agent-api-key", - avatar: 'https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png', + avatar: 'https://your-cdn/avatar.png', title: 'Get AI assistance', description: "DocsGPT's AI Chatbot is here to help", heroTitle: 'Welcome to DocsGPT!', @@ -143,7 +143,7 @@ The DocsGPT Widget offers a range of customizable properties that allow you to t |--------------------|------------------|-------------------------------------------------------------|-----------------------------------------------------------------------------------------------------| | **`apiHost`** | `string` | `"https://gptcloud.arc53.com"` | **Required.** The URL of your DocsGPT API backend. This endpoint handles vector search and chatbot queries. | | **`apiKey`** | `string` | `"your-api-key"` | API key for authentication with your DocsGPT API. Leave empty if no authentication is required. | -| **`avatar`** | `string` | [`dino-icon-link`](https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png) | URL for the avatar image displayed in the chatbot interface. | +| **`avatar`** | `string` | built-in placeholder | URL for the avatar image shown in the chatbot header. Defaults to a neutral silhouette drawn inline, so no image is fetched unless you supply one. | | **`title`** | `string` | `"Get AI assistance"` | Title text shown in the chatbot header. | | **`description`** | `string` | `"DocsGPT's AI Chatbot is here to help"` | Sub-title or descriptive text displayed below the title in the chatbot header. | | **`heroTitle`** | `string` | `"Welcome to DocsGPT !"` | Welcome message displayed when the chatbot is initially opened. | @@ -153,6 +153,32 @@ The DocsGPT Widget offers a range of customizable properties that allow you to t | **`buttonBg`** | `string` | `"#222327"` | Background color of the widget's launch button. | | **`size`** | `"small" \| "medium"` | `"medium"` | Size of the widget. Options: `"small"` or `"medium"`. Defaults to `"medium"`. | | **`showSources`** | `boolean` | `false` | Enables displaying source URLs for data fetched within the widget. When set to `true`, the widget will show the original sources of the fetched data. | +| **`allowedFileExtensions`** | `string[]` | _unset_ | File extensions the composer accepts, e.g. `['.pdf', '.docx', '.md', '.png']`. Setting it turns on attachments: an Attach button, drag-and-drop onto the panel, and paste-to-attach. Leave it unset to keep attachments off. | +| **`showMicButton`** | `boolean` | `false` | Adds a microphone that dictates a question into the input using the browser's Web Speech API. No backend STT provider needed; hides itself where the browser lacks the API. | + +--- + +## Attachments and Voice Input + +Both features are off until you turn them on, and each has a cost worth knowing about. + +**Attachments** are enabled by `allowedFileExtensions`, which acts as both the switch and the filter. Every attached file is uploaded to `/api/store_attachment`, parsed, and its text sent with the question, billed against the token budget of the key owner's account. Which types are worth that depends on your assistant, so there is no default list. A leading dot is optional (`'pdf'` works) and matching is case-insensitive. + +Files that finish uploading before the question is sent are attached to it, and the send waits while any are still uploading or parsing, so a file is never silently dropped. The list only filters what the picker offers: a file it allows but the backend cannot parse is still refused by the server, and the reason appears under that file's chip. + +**Voice input** is enabled by `showMicButton` and uses the browser's Web Speech API. Words appear in the input as they are spoken and extend whatever was already typed, so a question you had started typing survives. Nothing is uploaded to DocsGPT and no `STT_PROVIDER` is needed. + +It is off by default because of where the audio goes. Outside the Chromium builds that expose on-device recognition, the browser forwards the microphone to its vendor's speech service — Google's, in Chrome's case. That is a third-party data flow on your page, so it is your decision to enable. + +Browser support is uneven, and the microphone hides itself where it cannot work. Chrome, Edge and Safari have the API, Firefox does not enable it, and it requires a secure origin (HTTPS, or localhost). + +```jsx + +``` --- diff --git a/extensions/react-widget/README.md b/extensions/react-widget/README.md index 5b6222d2..d93ed110 100644 --- a/extensions/react-widget/README.md +++ b/extensions/react-widget/README.md @@ -29,7 +29,7 @@ To link the widget to your api and your documents you can pass parameters to the return ; }; ``` +`allowedFileExtensions` and `showMicButton` are both off unless you set them. Attached files are parsed and billed against your key's token budget, and the microphone uses the browser's Web Speech API, which forwards audio to the browser vendor's speech service outside on-device Chromium builds. See [the widget docs](https://docs.docsgpt.cloud/Extensions/chat-widget) for details and browser support. + ### Html ```html @@ -86,14 +90,16 @@ To link the widget to your api and your documents you can pass parameters to the renderDocsGPTWidget('app', { apiHost: 'http://localhost:7001', apiKey:"", - avatar: 'https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png', + avatar: 'https://your-cdn/avatar.png', title: 'Get AI assistance', description: "DocsGPT's AI Chatbot is here to help", heroTitle: 'Welcome to DocsGPT!', heroDescription: 'This chatbot is built with DocsGPT and utilises GenAI, please review important information using sources.', theme:"dark", buttonIcon:"https://your-icon.svg", - buttonBg:"#222327" + buttonBg:"#222327", + allowedFileExtensions: ['.pdf', '.docx', '.md', '.png'], + showMicButton: true }); } diff --git a/extensions/react-widget/src/App.tsx b/extensions/react-widget/src/App.tsx index 45567031..b30249e3 100644 --- a/extensions/react-widget/src/App.tsx +++ b/extensions/react-widget/src/App.tsx @@ -5,7 +5,10 @@ export const App = () => { return (
- +
); }; diff --git a/extensions/react-widget/src/components/ComposerControls.tsx b/extensions/react-widget/src/components/ComposerControls.tsx new file mode 100644 index 00000000..fb92fa30 --- /dev/null +++ b/extensions/react-widget/src/components/ComposerControls.tsx @@ -0,0 +1,623 @@ +import React from 'react'; +import styled, { keyframes, useTheme } from 'styled-components'; + +import { Attachment } from '../types/index'; +import { radii } from './tokens'; + +const ClipIcon = (props: React.SVGProps) => ( + + + +); + +const MicIcon = (props: React.SVGProps) => ( + + + + +); + +const SquareIcon = (props: React.SVGProps) => ( + + + +); + +const DocumentIcon = (props: React.SVGProps) => ( + + + + +); + +const AlertIcon = (props: React.SVGProps) => ( + + + + +); + +const CrossIcon = (props: React.SVGProps) => ( + + + +); + +const spin = keyframes` + to { transform: rotate(360deg); } +`; + +/** + * Determinate while bytes move, indeterminate while the server parses: that + * phase has no percentage, and a bar frozen at 100% reads as a hang. + */ +const ProgressRing = styled.svg<{ $indeterminate?: boolean }>` + width: 14px; + height: 14px; + transform: rotate(-90deg); + animation: ${(props) => (props.$indeterminate ? spin : 'none')} 900ms linear + infinite; + transform-origin: center; + + @media (prefers-reduced-motion: reduce) { + animation: none; + } +`; + +const CIRCUMFERENCE = 2 * Math.PI * 6; + +const AttachmentProgress = ({ attachment }: { attachment: Attachment }) => { + const indeterminate = attachment.status === 'processing'; + const fraction = indeterminate + ? 0.25 + : Math.min(Math.max(attachment.progress, 0), 100) / 100; + + return ( + + + + + ); +}; + +const ChipList = styled.div` + display: flex; + flex-wrap: wrap; + gap: 6px; + padding: 2px 2px 0 2px; +`; + +const FailureList = styled.div` + display: flex; + flex-direction: column; + gap: 2px; + padding: 4px 2px 0 2px; + font-size: 11.5px; + line-height: 1.45; + color: ${(props) => props.theme.danger!.text}; + overflow-wrap: anywhere; +`; + +const Chip = styled.div<{ $failed?: boolean }>` + box-sizing: border-box; + display: inline-flex; + align-items: center; + gap: 6px; + max-width: min(100%, 200px); + padding: 4px 4px 4px 8px; + border-radius: ${radii.full}; + font-size: 12px; + line-height: 1.5; + border: 1px solid + ${(props) => + props.$failed ? props.theme.danger!.border : props.theme.hairline}; + background: ${(props) => + props.$failed ? props.theme.danger!.soft : props.theme.primary.bg}; + color: ${(props) => + props.$failed ? props.theme.danger!.text : props.theme.primary.text}; +`; + +const ChipGlyph = styled.span` + display: inline-flex; + flex-shrink: 0; + align-items: center; + justify-content: center; + color: ${(props) => props.theme.accent!.base}; +`; + +const ChipLabel = styled.span` + min-width: 0; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; +`; + +const ChipRemove = styled.button` + display: inline-flex; + flex-shrink: 0; + align-items: center; + justify-content: center; + width: 18px; + height: 18px; + padding: 0; + border: none; + border-radius: ${radii.full}; + background: transparent; + color: inherit; + opacity: 0.65; + cursor: pointer; + transition: opacity 0.15s ease; + + &:hover { + opacity: 1; + } + + &:focus-visible { + outline: 2px solid ${(props) => props.theme.accent!.base}; + outline-offset: 1px; + opacity: 1; + } +`; + +/** Chip status, for the hover title. */ +const statusLabel = (attachment: Attachment): string => { + if (attachment.status === 'uploading') + return `Uploading ${attachment.progress}%`; + if (attachment.status === 'processing') return 'Processing'; + if (attachment.status === 'failed') return attachment.error ?? 'Failed'; + return 'Ready'; +}; + +export const AttachmentChips = ({ + attachments, + onRemove, +}: { + attachments: Attachment[]; + onRemove: (id: string) => void; +}) => { + // A touch user cannot see a tooltip, and a phone picker's unsupported + // file lands here. Say why in the open. + const failures = attachments.filter( + (attachment) => attachment.status === 'failed' && attachment.error, + ); + + return ( + <> + + {attachments.map((attachment) => { + const failed = attachment.status === 'failed'; + return ( + + + {attachment.fileName} + onRemove(attachment.id)} + aria-label={`Remove ${attachment.fileName}`} + > + + + + ); + })} + + + {failures.length > 0 && ( + + {failures.map((attachment) => ( + + {attachment.fileName}: {attachment.error} + + ))} + + )} + + ); +}; + +export const ControlBar = styled.div` + display: flex; + align-items: center; + gap: 6px; + padding: 0 2px; +`; + +export const ControlGroup = styled.div` + display: flex; + flex: 1; + min-width: 0; + flex-wrap: wrap; + align-items: center; + gap: 6px; +`; + +const ControlButton = styled.button<{ $recording?: boolean }>` + display: inline-flex; + align-items: center; + gap: 5px; + height: 28px; + padding: 0 10px; + border-radius: ${radii.full}; + border: 1px solid + ${(props) => + props.$recording ? props.theme.danger!.border : props.theme.hairline}; + background: ${(props) => + props.$recording ? props.theme.danger!.soft : 'transparent'}; + color: ${(props) => + props.$recording ? props.theme.danger!.text : props.theme.secondary.text}; + font-family: inherit; + font-size: 12px; + font-weight: 500; + line-height: 1; + cursor: pointer; + transition: + background-color 0.15s ease, + color 0.15s ease, + border-color 0.15s ease; + + &:hover:not(:disabled) { + background: ${(props) => + props.$recording ? props.theme.danger!.soft : props.theme.secondary.bg}; + color: ${(props) => + props.$recording ? props.theme.danger!.text : props.theme.primary.text}; + } + + &:focus-visible { + outline: 2px solid ${(props) => props.theme.accent!.base}; + outline-offset: 2px; + } + + &:disabled { + opacity: 0.5; + cursor: default; + } +`; + +export const AttachButton = ({ + onClick, + disabled, +}: { + onClick: () => void; + disabled?: boolean; +}) => ( + // A button opening a hidden input, not a label wrapping one: a + // `display: none` input takes no focus, so the label form is unreachable + // from the keyboard. + + +); + +export type MicButtonState = 'idle' | 'recording' | 'transcribing'; + +const MIC_LABELS: Record = { + idle: 'Voice', + recording: 'Stop', + transcribing: 'Transcribing', +}; + +const MIC_TITLES: Record = { + idle: 'Voice input', + recording: 'Stop recording', + transcribing: 'Transcribing audio', +}; + +export const MicButton = ({ + state, + disabled, + onClick, +}: { + state: MicButtonState; + disabled?: boolean; + onClick: () => void; +}) => ( + + {state === 'recording' ? ( + +); + +export const ComposerNote = styled.div<{ $tone?: 'danger' }>` + padding: 2px 4px 0 4px; + font-size: 11.5px; + line-height: 1.5; + color: ${(props) => + props.$tone === 'danger' + ? props.theme.danger!.text + : props.theme.secondary.text}; +`; + +const SentList = styled.div` + display: flex; + flex-wrap: wrap; + justify-content: flex-end; + gap: 6px; +`; + +const SentChip = styled.span` + display: inline-flex; + align-items: center; + gap: 5px; + max-width: min(100%, 200px); + padding: 3px 10px; + border-radius: ${radii.full}; + border: 1px solid ${(props) => props.theme.hairline}; + background: ${(props) => props.theme.secondary.bg}; + color: ${(props) => props.theme.secondary.text}; + font-size: 11.5px; + line-height: 1.6; +`; + +export const SentAttachments = ({ + attachments, +}: { + attachments: { id: string; fileName: string }[]; +}) => ( + + {attachments.map((attachment) => ( + + + ))} + +); + +// One bar per animation frame, so the row holds about a second of history. +const WAVEFORM_BARS = 48; +const BAR_GAP_RATIO = 0.4; +// Speech peaks well below full scale; the floor keeps a quiet line visible. +const LEVEL_GAIN = 2.8; +const MIN_BAR_RATIO = 0.06; + +const WaveformRow = styled.div` + display: flex; + flex: 1; + min-width: 0; + align-items: center; + gap: 10px; + padding: 0 10px; + min-height: ${(props) => + props.theme.dimensions!.size === 'large' ? '60px' : '40px'}; +`; + +const WaveformCanvas = styled.canvas` + flex: 1; + min-width: 0; + height: 28px; + display: block; +`; + +const ListeningLabel = styled.span` + flex-shrink: 0; + font-size: 11.5px; + font-weight: 500; + color: ${(props) => props.theme.secondary.text}; +`; + +/** + * Live microphone level, standing in for the input while dictation runs. + * Canvas inside a rAF loop, because sixty React updates a second would + * re-render the whole composer. A null analyser draws the resting line. + */ +export const VoiceWaveform = ({ + analyserRef, + label, +}: { + analyserRef: React.RefObject; + label: string; +}) => { + const canvasRef = React.useRef(null); + const theme = useTheme(); + const barColor = theme.accent!.base; + + React.useEffect(() => { + const canvas = canvasRef.current; + const context = canvas?.getContext('2d'); + if (!canvas || !context) return; + + const levels = new Array(WAVEFORM_BARS).fill(0); + // Bare `Uint8Array` widens to ArrayBufferLike, which the analyser + // signature rejects. + let samples: Uint8Array | null = null; + let frame = 0; + + const draw = () => { + frame = window.requestAnimationFrame(draw); + + const analyser = analyserRef.current; + let level = 0; + if (analyser) { + if (!samples || samples.length !== analyser.fftSize) + samples = new Uint8Array(analyser.fftSize); + analyser.getByteTimeDomainData(samples); + // Time-domain bytes ride on 128; RMS of the deviation is the level. + let sumSquares = 0; + for (let index = 0; index < samples.length; index += 1) { + const deviation = (samples[index] - 128) / 128; + sumSquares += deviation * deviation; + } + level = Math.sqrt(sumSquares / samples.length); + } + levels.push(Math.min(1, level * LEVEL_GAIN)); + levels.shift(); + + // Re-read each frame: the panel resizes and the row reflows. + const ratio = window.devicePixelRatio || 1; + const width = canvas.clientWidth; + const height = canvas.clientHeight; + if (!width || !height) return; + if (canvas.width !== Math.round(width * ratio)) { + canvas.width = Math.round(width * ratio); + canvas.height = Math.round(height * ratio); + } + context.setTransform(ratio, 0, 0, ratio, 0, 0); + context.clearRect(0, 0, width, height); + context.fillStyle = barColor; + + const slot = width / WAVEFORM_BARS; + const barWidth = Math.max(1, slot * (1 - BAR_GAP_RATIO)); + const radius = barWidth / 2; + for (let index = 0; index < WAVEFORM_BARS; index += 1) { + const magnitude = Math.max(MIN_BAR_RATIO, levels[index]); + const barHeight = magnitude * height; + const x = index * slot + (slot - barWidth) / 2; + const y = (height - barHeight) / 2; + context.globalAlpha = 0.35 + 0.65 * (index / WAVEFORM_BARS); + context.beginPath(); + context.roundRect(x, y, barWidth, barHeight, radius); + context.fill(); + } + context.globalAlpha = 1; + }; + + frame = window.requestAnimationFrame(draw); + return () => window.cancelAnimationFrame(frame); + }, [analyserRef, barColor]); + + return ( + + + ); +}; + +export const DropOverlay = styled.div` + position: absolute; + inset: 0; + z-index: 5; + display: flex; + align-items: center; + justify-content: center; + pointer-events: none; + padding: 16px; + box-sizing: border-box; + background: ${(props) => props.theme.primary.bg}; + opacity: 0.94; + font-size: 13px; + font-weight: 500; + color: ${(props) => props.theme.secondary.text}; +`; + +export const DropTarget = styled.div` + display: flex; + align-items: center; + gap: 8px; + padding: 18px 22px; + border-radius: ${radii.md}; + border: 1px dashed ${(props) => props.theme.accent!.base}; + background: ${(props) => props.theme.accent!.soft}; + color: ${(props) => props.theme.primary.text}; +`; + +export { ClipIcon }; diff --git a/extensions/react-widget/src/components/DocsGPTWidget.tsx b/extensions/react-widget/src/components/DocsGPTWidget.tsx index 91fa16f1..a6a1bc97 100644 --- a/extensions/react-widget/src/components/DocsGPTWidget.tsx +++ b/extensions/react-widget/src/components/DocsGPTWidget.tsx @@ -14,11 +14,34 @@ import { FEEDBACK, MESSAGE_TYPE, Query, + SentAttachment, Status, WidgetCoreProps, WidgetProps, } from '../types/index'; import { fetchAnswerStreaming, sendFeedback } from '../requests/streamingApi'; +import { + acceptAttribute, + normalizeExtensions, + useAttachments, +} from '../hooks/useAttachments'; +import { useVoiceInput, voiceInputSupported } from '../hooks/useVoiceInput'; +import { + AttachButton, + AttachmentChips, + ClipIcon, + ComposerNote, + ControlBar, + ControlGroup, + DropOverlay, + DropTarget, + MicButton, + type MicButtonState, + SentAttachments, + VoiceWaveform, +} from './ComposerControls'; +import { DEFAULT_AVATAR } from './defaultAvatar'; +import { radii } from './tokens'; import { ThemeProvider } from 'styled-components'; import MarkdownIt from 'markdown-it'; import { @@ -222,14 +245,6 @@ const themes = { }, }; -const radii = { - sm: '8px', - md: '12px', - lg: '18px', - panel: '16px', - full: '9999px', -}; - const sizesConfig = { small: { size: 'small', width: '320px', height: '400px' }, medium: { size: 'medium', width: '400px', height: '80vh' }, @@ -1007,17 +1022,20 @@ const Composer = styled.div` padding: 12px 16px 0 16px; border-top: 1px solid ${(props) => props.theme.hairline}; `; -const PromptContainer = styled.form` +const PromptContainer = styled.form<{ $stacked?: boolean }>` box-sizing: border-box; - padding: 4px 4px 4px 6px; + padding: ${(props) => + props.$stacked ? '6px 8px 8px 8px' : '4px 4px 4px 6px'}; background-color: ${(props) => props.theme.secondary.bg}; border: 1px solid ${(props) => props.theme.hairline}; - border-radius: 24px; + border-radius: ${(props) => (props.$stacked ? radii.lg : '24px')}; min-height: ${(props) => props.theme.dimensions!.size == 'large' ? '40px' : '23px'}; - max-height: 150px; + /* Stacked needs room for the chips and control row; inline keeps the cap. */ + max-height: ${(props) => (props.$stacked ? 'none' : '150px')}; display: flex; - align-items: end; + flex-direction: ${(props) => (props.$stacked ? 'column' : 'row')}; + align-items: ${(props) => (props.$stacked ? 'stretch' : 'end')}; gap: 6px; transition: border-color 0.15s ease, @@ -1028,8 +1046,20 @@ const PromptContainer = styled.form` box-shadow: 0 0 0 3px ${(props) => props.theme.accent!.soft}; } `; -const StyledTextarea = styled.textarea` +const PromptRow = styled.div` box-sizing: border-box; + display: flex; + align-items: end; + gap: 6px; + width: 100%; + min-width: 0; +`; +const HiddenFileInput = styled.input` + display: none; +`; +const StyledTextarea = styled.textarea<{ $hidden?: boolean }>` + box-sizing: border-box; + ${(props) => (props.$hidden ? 'display: none;' : '')} width: 100%; border: none; padding: ${(props) => @@ -1151,10 +1181,14 @@ const HeroDescription = styled.p` padding: 0px; `; const Hyperlink = styled.a` - color: ${(props) => props.theme.accent!.link}; - text-decoration: none; + /* Inherits the tagline colour; the underline is what marks it as a link. */ + color: inherit; + text-decoration: underline; + /* Keeps descenders clear of the rule at 11px. */ + text-underline-offset: 2px; + transition: opacity 0.2s ease; &:hover { - text-decoration: underline; + opacity: 0.8; } `; const Tagline = styled.div` @@ -1302,7 +1336,7 @@ export const DocsGPTWidget = (props: WidgetProps) => { export const WidgetCore = ({ apiHost = 'https://gptcloud.arc53.com', apiKey = '527686a3-e867-4b4d-9fec-f5f45fdb613a', - avatar = 'https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png', + avatar = DEFAULT_AVATAR, title = 'Get AI assistance', description = "DocsGPT's AI Chatbot is here to help", heroTitle = 'Welcome to DocsGPT !', @@ -1314,6 +1348,8 @@ export const WidgetCore = ({ showSources = true, handleClose, prefilledQuery = '', + allowedFileExtensions, + showMicButton = false, }: WidgetCoreProps) => { const [prompt, setPrompt] = React.useState(''); const [mounted, setMounted] = React.useState(false); @@ -1329,11 +1365,85 @@ export const WidgetCore = ({ const [feedbackErrorIndex, setFeedbackErrorIndex] = React.useState< number | null >(null); + const [isDraggingFiles, setIsDraggingFiles] = React.useState(false); const abortRef = useRef(null); const conversationRef = useRef(null); const endMessageRef = React.useRef(null); const promptRef = React.useRef(null); + const attachmentInputRef = React.useRef(null); + // The draft as it stood when dictation began; recognised words extend it. + const voiceBaseRef = React.useRef(''); + // dragenter/dragleave fire per child crossed, hence a depth count. + const dragDepthRef = React.useRef(0); + + // The list doubles as the on switch: no accepted types, no attachments. + const acceptedExtensions = React.useMemo( + () => normalizeExtensions(allowedFileExtensions), + [allowedFileExtensions], + ); + const attachmentsEnabled = acceptedExtensions.length > 0; + + const { + attachments, + addFiles, + remove: removeAttachment, + clear: clearAttachments, + pendingCount, + failedCount, + completed: completedAttachments, + } = useAttachments({ apiKey, apiHost, acceptedExtensions }); + + // One place for the height arithmetic: typing, transcripts and sends all + // go through here. + const resizePrompt = React.useCallback(() => { + const el = promptRef.current; + if (!el) return; + const baseHeight = size === 'large' ? 60 : 40; + const maxHeight = 140; + el.style.height = 'auto'; + el.style.height = `${Math.max( + baseHeight, + Math.min(el.scrollHeight, maxHeight), + )}px`; + }, [size]); + + // The textarea is controlled by `prompt`, so its value is the live draft. + const handleVoiceStart = React.useCallback(() => { + voiceBaseRef.current = promptRef.current?.value ?? ''; + }, []); + + const applyTranscript = React.useCallback( + (text: string) => { + // The base is fixed at the start, so revised interim words overwrite + // only themselves and a typed half-question survives. + const base = voiceBaseRef.current; + setPrompt(base.trim() ? `${base.replace(/\s+$/, '')}\n${text}` : text); + // The height can only be recomputed once React has committed. + window.requestAnimationFrame(resizePrompt); + }, + [resizePrompt], + ); + + const handleVoiceEnd = React.useCallback(() => { + // Once at the end, not on every interim word. + window.requestAnimationFrame(() => promptRef.current?.focus()); + }, []); + + const { + recordingState, + error: voiceError, + toggle: toggleVoiceInput, + clearError: clearVoiceError, + analyserRef: voiceAnalyserRef, + } = useVoiceInput({ + onStart: handleVoiceStart, + onTranscript: applyTranscript, + onEnd: handleVoiceEnd, + }); + + // Firefox has no SpeechRecognition, nor does an insecure origin. + const canUseVoice = showMicButton && voiceInputSupported(); const md = new MarkdownIt(); //Custom markdown for the table md.renderer.rules.table_open = () => @@ -1447,7 +1557,7 @@ export const WidgetCore = ({ setStatus('idle'); }; - async function stream(question: string) { + async function stream(question: string, attachmentIds: string[] = []) { setStatus('loading'); const controller = new AbortController(); abortRef.current = controller; @@ -1459,6 +1569,7 @@ export const WidgetCore = ({ apiHost: apiHost, history: queries, conversationId: conversationId, + attachments: attachmentIds, onEvent: (event: MessageEvent) => { let data: StreamEvent; try { @@ -1562,10 +1673,24 @@ export const WidgetCore = ({ const appendQuery = async (userQuery: string) => { if (!userQuery) return; + // Only parsed attachments have a server id. The send is held until none + // are pending, so this is the whole list. + const sent: SentAttachment[] = completedAttachments.map((attachment) => ({ + id: attachment.attachmentId as string, + fileName: attachment.fileName, + })); + setIsPinnedToLatest(true); - queries.push({ prompt: userQuery }); + queries.push({ + prompt: userQuery, + attachments: sent.length > 0 ? sent : undefined, + }); setPrompt(''); - await stream(userQuery); + if (sent.length > 0) clearAttachments(); + await stream( + userQuery, + sent.map((attachment) => attachment.id), + ); }; const handleCopy = async (text: string, index: number) => { try { @@ -1585,48 +1710,59 @@ export const WidgetCore = ({ if (status === 'loading') return; const prompt = queries[index]?.prompt; if (!prompt) return; + // The composer list was cleared on send, so the row's ids are the only + // record of what the question carried. + const attached = queries[index]?.attachments; setQueries((prev: Query[]) => { const updated = [...prev]; - updated[index] = { prompt }; + updated[index] = { prompt, attachments: attached }; return updated.slice(0, index + 1); }); setIsPinnedToLatest(true); - await stream(prompt); + await stream( + prompt, + (attached ?? []).map((attachment) => attachment.id), + ); + }; + + // Pending and failed attachments both hold the send, so neither is + // silently dropped. Pending reads first: it clears on its own. + const pendingNote = + pendingCount > 0 + ? `Waiting for ${pendingCount} file${pendingCount === 1 ? '' : 's'} to finish\u2026` + : null; + const failedNote = + failedCount > 0 + ? 'Remove the file that could not be attached, then send.' + : null; + const sendBlockedReason = pendingNote ?? failedNote; + const isDictating = + recordingState === 'recording' || recordingState === 'transcribing'; + const canSubmit = + prompt.trim().length > 0 && !sendBlockedReason && !isDictating; + + const submitPrompt = async () => { + if (!canSubmit) return; + // Before the value clears, or the composer sits tall and empty a frame. + if (promptRef.current) promptRef.current.style.height = 'auto'; + await appendQuery(prompt); }; - // submit handler const handleSubmit = async (e: React.FormEvent) => { e.preventDefault(); - if (!prompt.trim()) return; - if (promptRef.current) { - promptRef.current.style.height = 'auto'; - } - await appendQuery(prompt); + await submitPrompt(); }; const handlePromptKeyDown = async ( e: React.KeyboardEvent, ) => { if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); - // Prevent sending empty messages - if (promptRef.current && promptRef.current.value.trim() === '') return; - //Rest the input to it's original size after submitting - if (promptRef.current) { - promptRef.current.value = ''; - promptRef.current.style.height = 'auto'; - } - await appendQuery(prompt); + await submitPrompt(); } }; // Auto-resize the input textarea while typing, clamping to base or max height const handleUserInput = () => { - const el = promptRef.current; - if (!el) return; - const baseHeight = size === 'large' ? 60 : 40; - const maxHeight = 140; - el.style.height = 'auto'; - const next = Math.min(el.scrollHeight, maxHeight); - el.style.height = Math.max(baseHeight, next) + 'px'; + resizePrompt(); }; // Update prompt state, auto resize textarea to content, and maintain scroll on new lines @@ -1634,22 +1770,78 @@ export const WidgetCore = ({ event: React.ChangeEvent, ) => { const value = event.target.value; + // A stale voice error would hide the attachment notice below. + if (voiceError) clearVoiceError(); setPrompt(value); - const el = event.currentTarget; - const baseHeight = size === 'large' ? 60 : 40; - const maxHeight = 140; - el.style.height = 'auto'; - const next = Math.min(el.scrollHeight, maxHeight); - el.style.height = Math.max(baseHeight, next) + 'px'; + resizePrompt(); if (value.includes('\n')) { + const el = event.currentTarget; el.scrollTop = el.scrollHeight; } }; + + const handleAttachmentPick = (e: React.ChangeEvent) => { + const files = Array.from(e.target.files ?? []); + // Clear it or picking the same file twice fires no change event. + e.target.value = ''; + if (files.length > 0) addFiles(files); + }; + + const handlePromptPaste = (e: React.ClipboardEvent) => { + if (!attachmentsEnabled || isDictating) return; + const items = e.clipboardData?.items; + if (!items) return; + const files: File[] = []; + for (let index = 0; index < items.length; index += 1) { + if (items[index].kind !== 'file') continue; + const file = items[index].getAsFile(); + if (file) files.push(file); + } + if (files.length === 0) return; + // Or the file also lands in the textarea as binary noise. + e.preventDefault(); + addFiles(files); + }; + + // Otherwise the overlay flashes on every drag crossing the page. + const isFileDrag = (e: React.DragEvent) => + Array.from(e.dataTransfer?.types ?? []).includes('Files'); + + // A file dropped mid-dictation would queue behind the waveform, unseen + // and unsendable until recording stops. + const acceptsFiles = attachmentsEnabled && !isDictating; + + const handleDragEnter = (e: React.DragEvent) => { + if (!acceptsFiles || !isFileDrag(e)) return; + dragDepthRef.current += 1; + setIsDraggingFiles(true); + }; + + const handleDragOver = (e: React.DragEvent) => { + if (!acceptsFiles || !isFileDrag(e)) return; + // Without this the browser opens the file instead of dropping it. + e.preventDefault(); + e.dataTransfer.dropEffect = 'copy'; + }; + + const handleDragLeave = (e: React.DragEvent) => { + if (!acceptsFiles || !isFileDrag(e)) return; + dragDepthRef.current = Math.max(0, dragDepthRef.current - 1); + if (dragDepthRef.current === 0) setIsDraggingFiles(false); + }; + + const handleDrop = (e: React.DragEvent) => { + if (!acceptsFiles) return; + e.preventDefault(); + dragDepthRef.current = 0; + setIsDraggingFiles(false); + const files = Array.from(e.dataTransfer?.files ?? []); + if (files.length > 0) addFiles(files); + }; const handleImageError = ( event: React.SyntheticEvent, ) => { - event.currentTarget.src = - 'https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png'; + event.currentTarget.src = DEFAULT_AVATAR; }; const renderStatusLine = (query: Query, index: number) => { @@ -1667,6 +1859,43 @@ export const WidgetCore = ({ ); }; + // Neither feature enabled keeps the original single-row composer. + const hasComposerControls = attachmentsEnabled || canUseVoice; + + const micButtonState: MicButtonState = + recordingState === 'recording' || recordingState === 'transcribing' + ? recordingState + : 'idle'; + + // A failure to act on outranks a wait that clears itself. + const composerNote = voiceError + ? { text: voiceError, tone: 'danger' as const } + : sendBlockedReason + ? { + text: sendBlockedReason, + tone: pendingNote ? undefined : ('danger' as const), + } + : null; + + const sendControl = + status === 'loading' ? ( + + + + ) : ( + + + + ); + const baseDimensions = typeof size === 'object' && 'custom' in size ? sizesConfig.getCustom(size.custom) @@ -1686,7 +1915,13 @@ export const WidgetCore = ({ className={`${size !== 'large' ? (isOpen ? 'open' : 'close') : 'modal'}`} $modal={size === 'large'} > - +
@@ -1729,6 +1964,12 @@ export const WidgetCore = ({ {query.prompt && ( + {query.attachments && + query.attachments.length > 0 && ( + + )} - - - {status === 'loading' ? ( - - - - ) : ( - - - + )} + + {attachments.length > 0 && ( + + )} + + {isDictating && ( + + )} + {/* Hidden, not unmounted: the ref stays live for reading + the draft and refocusing afterwards. */} + + {!hasComposerControls && sendControl} + + {hasComposerControls && ( + + + {attachmentsEnabled && ( + attachmentInputRef.current?.click()} + disabled={isDictating} + /> + )} + {canUseVoice && ( + + )} + + {sendControl} + )} + {composerNote && ( + + {composerNote.text} + + )} Powered by  @@ -1939,6 +2234,14 @@ export const WidgetCore = ({ + {isDraggingFiles && ( + + + + + )} } diff --git a/extensions/react-widget/src/components/SearchBar.tsx b/extensions/react-widget/src/components/SearchBar.tsx index a6a8ae18..f3e6b6ea 100644 --- a/extensions/react-widget/src/components/SearchBar.tsx +++ b/extensions/react-widget/src/components/SearchBar.tsx @@ -1,6 +1,7 @@ import React from 'react'; import styled, { ThemeProvider, createGlobalStyle } from 'styled-components'; import { WidgetCore } from './DocsGPTWidget'; +import { DEFAULT_AVATAR } from './defaultAvatar'; import { SearchBarProps } from '@/types'; import { getSearchResults } from '../requests/searchAPI'; import { Result } from '@/types'; @@ -491,12 +492,7 @@ export const SearchBar = ({ - DocsGPT + Ask the AI diff --git a/extensions/react-widget/src/components/defaultAvatar.ts b/extensions/react-widget/src/components/defaultAvatar.ts new file mode 100644 index 00000000..f7337f98 --- /dev/null +++ b/extensions/react-widget/src/components/defaultAvatar.ts @@ -0,0 +1,12 @@ +const AVATAR_SVG = ` + + + + + + +`; + +export const DEFAULT_AVATAR = `data:image/svg+xml,${encodeURIComponent( + AVATAR_SVG, +)}`; diff --git a/extensions/react-widget/src/components/tokens.ts b/extensions/react-widget/src/components/tokens.ts new file mode 100644 index 00000000..097cf678 --- /dev/null +++ b/extensions/react-widget/src/components/tokens.ts @@ -0,0 +1,8 @@ +/** Corner radii shared by the widget's chrome. */ +export const radii = { + sm: '8px', + md: '12px', + lg: '18px', + panel: '16px', + full: '9999px', +}; diff --git a/extensions/react-widget/src/hooks/useAttachments.ts b/extensions/react-widget/src/hooks/useAttachments.ts new file mode 100644 index 00000000..24f59da9 --- /dev/null +++ b/extensions/react-widget/src/hooks/useAttachments.ts @@ -0,0 +1,224 @@ +import React from 'react'; + +import { fetchTaskOutcome, uploadAttachment } from '../requests/attachmentsApi'; +import { Attachment } from '../types/index'; + +const POLL_INTERVAL_MS = 1500; +// A large PDF with OCR can take minutes to parse. +const POLL_TIMEOUT_MS = 5 * 60 * 1000; + +const PROCESSING_FAILED = 'Could not read this file.'; + +const generateId = (): string => + `${Date.now()}-${Math.random().toString(36).slice(2)}`; + +/** Lower-cased final extension including the dot, or '' (dotfiles have none). */ +const fileExtension = (name: string): string => { + const base = name.split(/[\\/]/).pop() ?? ''; + const dot = base.lastIndexOf('.'); + return dot <= 0 ? '' : base.slice(dot).toLowerCase(); +}; + +/** `allowedFileExtensions` as '.ext' entries; 'pdf' and '.PDF' both work. */ +export const normalizeExtensions = (types?: string[]): string[] => { + if (!Array.isArray(types)) return []; + return types + .map((type) => String(type).trim().toLowerCase()) + .filter(Boolean) + .map((type) => (type.startsWith('.') ? type : `.${type}`)); +}; + +/** Picker filter; a hint only, `addFiles` does the real check. */ +export const acceptAttribute = (extensions: string[]): string => + extensions.join(','); + +/** Resolves on the timeout, or early on abort. */ +const delay = (ms: number, signal: AbortSignal): Promise => + new Promise((resolve) => { + const onAbort = () => { + clearTimeout(timer); + resolve(); + }; + const timer = setTimeout(() => { + signal.removeEventListener('abort', onAbort); + resolve(); + }, ms); + signal.addEventListener('abort', onAbort, { once: true }); + }); + +interface UseAttachmentsOptions { + apiKey: string; + apiHost: string; + /** Normalised '.ext' list; empty accepts nothing. */ + acceptedExtensions: string[]; +} + +/** + * Upload each picked file, then poll its parse task for an attachment id. + * + * The widget authenticates on an agent key, so it has no user event stream + * to subscribe to and completion has to be polled. + */ +export const useAttachments = ({ + apiKey, + apiHost, + acceptedExtensions, +}: UseAttachmentsOptions) => { + const [attachments, setAttachments] = React.useState([]); + // One per chip, so removing it cancels the upload and stops the poll. + const controllersRef = React.useRef(new Map()); + + const patch = React.useCallback( + (id: string, updates: Partial) => { + // A removed chip is not matched, so a late response cannot resurrect + // it. + setAttachments((prev) => + prev.map((item) => (item.id === id ? { ...item, ...updates } : item)), + ); + }, + [], + ); + + const track = React.useCallback( + async (id: string, taskId: string, controller: AbortController) => { + const deadline = Date.now() + POLL_TIMEOUT_MS; + + while (!controller.signal.aborted) { + await delay(POLL_INTERVAL_MS, controller.signal); + if (controller.signal.aborted) return; + + let outcome; + try { + outcome = await fetchTaskOutcome(taskId, apiHost, controller.signal); + } catch { + // Offline for a moment; the deadline below bounds this. + outcome = { state: 'pending' as const }; + } + if (controller.signal.aborted) return; + + if (outcome.state === 'completed') { + // No id means nothing to send, whatever the task status said. + if (outcome.attachmentId) + patch(id, { + status: 'completed', + progress: 100, + attachmentId: outcome.attachmentId, + }); + else patch(id, { status: 'failed', error: PROCESSING_FAILED }); + return; + } + if (outcome.state === 'failed') { + patch(id, { status: 'failed', error: PROCESSING_FAILED }); + return; + } + if (Date.now() > deadline) { + patch(id, { + status: 'failed', + error: 'Took too long to process.', + }); + return; + } + } + }, + [apiHost, patch], + ); + + const addFiles = React.useCallback( + (files: File[]) => { + files.forEach((file) => { + const id = generateId(); + const extension = fileExtension(file.name); + + // A rejected file still gets a chip: mobile pickers ignore + // `accept`, and a silent drop looks broken. + if (!acceptedExtensions.includes(extension)) { + setAttachments((prev) => [ + ...prev, + { + id, + fileName: file.name, + status: 'failed', + progress: 0, + error: `${extension || 'That file type'} is not accepted here.`, + }, + ]); + return; + } + + const controller = new AbortController(); + controllersRef.current.set(id, controller); + setAttachments((prev) => [ + ...prev, + { id, fileName: file.name, status: 'uploading', progress: 0 }, + ]); + + uploadAttachment({ + file, + apiKey, + apiHost, + signal: controller.signal, + onProgress: (percent) => patch(id, { progress: percent }), + }) + .then(({ taskId }) => { + if (controller.signal.aborted) return undefined; + patch(id, { status: 'processing', progress: 100 }); + return track(id, taskId, controller); + }) + .catch((error: unknown) => { + if (controller.signal.aborted) return; + patch(id, { + status: 'failed', + error: + error instanceof Error && error.message + ? error.message + : 'Upload failed.', + }); + }) + .finally(() => { + controllersRef.current.delete(id); + }); + }); + }, + [acceptedExtensions, apiHost, apiKey, patch, track], + ); + + const remove = React.useCallback((id: string) => { + controllersRef.current.get(id)?.abort(); + controllersRef.current.delete(id); + setAttachments((prev) => prev.filter((item) => item.id !== id)); + }, []); + + const clear = React.useCallback(() => { + controllersRef.current.forEach((controller) => controller.abort()); + controllersRef.current.clear(); + setAttachments([]); + }, []); + + React.useEffect(() => { + const controllers = controllersRef.current; + return () => { + controllers.forEach((controller) => controller.abort()); + controllers.clear(); + }; + }, []); + + const pendingCount = attachments.filter( + (item) => item.status === 'uploading' || item.status === 'processing', + ).length; + const failedCount = attachments.filter( + (item) => item.status === 'failed', + ).length; + const completed = attachments.filter( + (item) => item.status === 'completed' && item.attachmentId, + ); + + return { + attachments, + addFiles, + remove, + clear, + pendingCount, + failedCount, + completed, + }; +}; diff --git a/extensions/react-widget/src/hooks/useVoiceInput.ts b/extensions/react-widget/src/hooks/useVoiceInput.ts new file mode 100644 index 00000000..516f48b3 --- /dev/null +++ b/extensions/react-widget/src/hooks/useVoiceInput.ts @@ -0,0 +1,245 @@ +import React from 'react'; + +//Recognition runs in-browser + +export type RecordingState = 'idle' | 'recording' | 'transcribing' | 'error'; + +interface SpeechRecognitionLike extends EventTarget { + continuous: boolean; + interimResults: boolean; + lang: string; + maxAlternatives: number; + start(): void; + stop(): void; + abort(): void; + onresult: ((event: SpeechRecognitionEvent) => void) | null; + onerror: ((event: SpeechRecognitionErrorEvent) => void) | null; + onend: (() => void) | null; +} + +type SpeechRecognitionConstructor = new () => SpeechRecognitionLike; + +type LegacyAudioWindow = Window & + typeof globalThis & { webkitAudioContext?: typeof AudioContext }; + +const LEVEL_FFT_SIZE = 1024; +const LEVEL_SMOOTHING = 0.6; + +type SpeechWindow = Window & + typeof globalThis & { + SpeechRecognition?: SpeechRecognitionConstructor; + webkitSpeechRecognition?: SpeechRecognitionConstructor; + }; + +const recognitionClass = (): SpeechRecognitionConstructor | undefined => { + if (typeof window === 'undefined') return undefined; + const speechWindow = window as SpeechWindow; + return speechWindow.SpeechRecognition ?? speechWindow.webkitSpeechRecognition; +}; + +export const voiceInputSupported = (): boolean => { + if (typeof window === 'undefined') return false; + // Gated on a secure origin, where the failure looks identical to the API + // being absent. + if (!window.isSecureContext) return false; + return recognitionClass() !== undefined; +}; + +/** Message worth showing, or null for outcomes that are not errors. */ +const errorMessage = (code: string): string | null => { + switch (code) { + // The user pressed stop. + case 'aborted': + return null; + case 'no-speech': + return 'No speech was picked up.'; + case 'not-allowed': + case 'service-not-allowed': + return 'Microphone access was blocked.'; + case 'audio-capture': + return 'No microphone was found.'; + case 'network': + return 'Voice input is unavailable right now.'; + case 'language-not-supported': + return 'Voice input does not support this language.'; + default: + return 'Voice input stopped unexpectedly.'; + } +}; + +interface UseVoiceInputOptions { + onStart: () => void; + /** Final words plus the interim tail. */ + onTranscript: (text: string) => void; + onEnd?: () => void; +} + +/** The composer's microphone; one button toggles it. */ +export const useVoiceInput = ({ + onStart, + onTranscript, + onEnd, +}: UseVoiceInputOptions) => { + const [recordingState, setRecordingState] = + React.useState('idle'); + const [error, setError] = React.useState(null); + const recognitionRef = React.useRef(null); + // The Web Speech API exposes no audio, so the waveform needs a second + // capture of its own. Kept in a ref so the draw loop re-renders nothing. + const analyserRef = React.useRef(null); + const audioContextRef = React.useRef(null); + const levelStreamRef = React.useRef(null); + // Finals accumulate; the interim tail is rebuilt each event, never kept. + const finalTranscriptRef = React.useRef(''); + const isMountedRef = React.useRef(true); + + const stopLevelMeter = React.useCallback(() => { + analyserRef.current = null; + levelStreamRef.current?.getTracks().forEach((track) => track.stop()); + levelStreamRef.current = null; + void audioContextRef.current?.close().catch(() => undefined); + audioContextRef.current = null; + }, []); + + /** Best-effort: a browser that refuses this still transcribes fine. */ + const startLevelMeter = React.useCallback(async () => { + const audioWindow = window as LegacyAudioWindow; + const AudioContextClass = + audioWindow.AudioContext ?? audioWindow.webkitAudioContext; + if (!AudioContextClass || !navigator.mediaDevices?.getUserMedia) return; + + // Never `await context.resume()`: without user activation that promise + // never settles at all, stranding everything below it, so no analyser is + // built. Constructed before the first await so it stays under the click's + // activation, which is insurance for browsers that do not treat a live + // capture as activation the way Chrome does. + const context = new AudioContextClass(); + + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + // Recording may have ended while permission was granted. + if (!isMountedRef.current || !recognitionRef.current) { + stream.getTracks().forEach((track) => track.stop()); + void context.close().catch(() => undefined); + return; + } + + const analyser = context.createAnalyser(); + analyser.fftSize = LEVEL_FFT_SIZE; + analyser.smoothingTimeConstant = LEVEL_SMOOTHING; + // Not connected to the destination: that would play the microphone + // back through the page's speakers. + context.createMediaStreamSource(stream).connect(analyser); + + levelStreamRef.current = stream; + audioContextRef.current = context; + analyserRef.current = analyser; + } catch { + void context.close().catch(() => undefined); + } + }, []); + + React.useEffect(() => { + isMountedRef.current = true; + return () => { + isMountedRef.current = false; + // `abort`, not `stop`: nothing left to deliver, and it frees the + // microphone at once. + recognitionRef.current?.abort(); + recognitionRef.current = null; + stopLevelMeter(); + }; + }, [stopLevelMeter]); + + const start = React.useCallback(() => { + const RecognitionClass = recognitionClass(); + if (!RecognitionClass) { + setRecordingState('error'); + setError( + typeof window !== 'undefined' && !window.isSecureContext + ? 'Voice input needs a secure connection (HTTPS).' + : 'This browser does not support voice input.', + ); + return; + } + + const recognition = new RecognitionClass(); + // Without this the engine stops at the first pause. + recognition.continuous = true; + recognition.interimResults = true; + recognition.maxAlternatives = 1; + if (typeof navigator !== 'undefined' && navigator.language) + recognition.lang = navigator.language; + + finalTranscriptRef.current = ''; + + recognition.onresult = (event: SpeechRecognitionEvent) => { + let interim = ''; + // From `resultIndex`, not 0: the API returns the whole list every + // time, and earlier results are already banked. + for ( + let index = event.resultIndex; + index < event.results.length; + index += 1 + ) { + const result = event.results[index]; + const text = result[0]?.transcript ?? ''; + if (result.isFinal) finalTranscriptRef.current += text; + else interim += text; + } + onTranscript(`${finalTranscriptRef.current}${interim}`.trim()); + }; + + recognition.onerror = (event: SpeechRecognitionErrorEvent) => { + const message = errorMessage(event.error); + if (!isMountedRef.current || !message) return; + setRecordingState('error'); + setError(message); + }; + + recognition.onend = () => { + recognitionRef.current = null; + // Not on stop(), so the bars stay live while the last utterance + // finalises. + stopLevelMeter(); + if (!isMountedRef.current) return; + // An error already set its own state; do not overwrite it. + setRecordingState((previous) => + previous === 'error' ? previous : 'idle', + ); + onEnd?.(); + }; + + try { + recognition.start(); + } catch { + // Already running, if a stop is still unwinding. + return; + } + + recognitionRef.current = recognition; + setError(null); + onStart(); + setRecordingState('recording'); + void startLevelMeter(); + }, [onEnd, onStart, onTranscript, startLevelMeter, stopLevelMeter]); + + const toggle = React.useCallback(() => { + if (recordingState === 'transcribing') return; + if (recordingState === 'recording') { + // `stop`, not `abort`: the last utterance still has a final result + // to deliver. + setRecordingState('transcribing'); + recognitionRef.current?.stop(); + return; + } + start(); + }, [recordingState, start]); + + const clearError = React.useCallback(() => { + setError(null); + setRecordingState((previous) => (previous === 'error' ? 'idle' : previous)); + }, []); + + return { recordingState, error, toggle, clearError, analyserRef }; +}; diff --git a/extensions/react-widget/src/requests/attachmentsApi.ts b/extensions/react-widget/src/requests/attachmentsApi.ts new file mode 100644 index 00000000..d9762bf9 --- /dev/null +++ b/extensions/react-widget/src/requests/attachmentsApi.ts @@ -0,0 +1,128 @@ +/** + * `/api/store_attachment` returns a Celery task id; the parsed attachment + * only exists once that task lands, hence the polling below. The id `/stream` + * expects is the task result's `attachment_id`, not the one the upload + * response carries. Auth is the agent key as `api_key` in the multipart body. + */ + +interface UploadAttachmentOptions { + file: File; + apiKey: string; + apiHost: string; + onProgress?: (percent: number) => void; + signal?: AbortSignal; +} + +export interface UploadedAttachment { + taskId: string; +} + +export type TaskOutcome = + | { state: 'pending' } + | { state: 'completed'; attachmentId?: string } + | { state: 'failed' }; + +/** The route composes these itself (too large, unsupported type), so they are safe to show. */ +const errorMessageFrom = (body: string): string | undefined => { + try { + const parsed = JSON.parse(body); + const message = parsed?.errors?.[0]?.error ?? parsed?.message; + return typeof message === 'string' && message ? message : undefined; + } catch { + return undefined; + } +}; + +export function uploadAttachment({ + file, + apiKey, + apiHost, + onProgress, + signal, +}: UploadAttachmentOptions): Promise { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(new DOMException('Upload aborted', 'AbortError')); + return; + } + + const formData = new FormData(); + formData.append('file', file); + if (apiKey) formData.append('api_key', apiKey); + + const xhr = new XMLHttpRequest(); + const onAbort = () => xhr.abort(); + signal?.addEventListener('abort', onAbort, { once: true }); + + // fetch cannot report upload progress. + xhr.upload.addEventListener('progress', (event) => { + if (event.lengthComputable) + onProgress?.(Math.round((event.loaded / event.total) * 100)); + }); + + xhr.onload = () => { + signal?.removeEventListener('abort', onAbort); + const fallback = errorMessageFrom(xhr.responseText); + if (xhr.status !== 200) { + reject(new Error(fallback ?? 'Upload failed.')); + return; + } + try { + const response = JSON.parse(xhr.responseText); + // One file answers with a bare `task_id`; the batch shape is read + // too, in case the route stops special-casing one file. + const task = response?.task_id ? response : response?.tasks?.[0]; + if (!task?.task_id) { + reject(new Error(fallback ?? 'Upload failed.')); + return; + } + resolve({ taskId: task.task_id }); + } catch { + reject(new Error('Upload failed.')); + } + }; + + xhr.onerror = () => { + signal?.removeEventListener('abort', onAbort); + reject(new Error('Upload failed.')); + }; + xhr.onabort = () => { + signal?.removeEventListener('abort', onAbort); + reject(new DOMException('Upload aborted', 'AbortError')); + }; + + xhr.open('POST', `${apiHost}/api/store_attachment`); + xhr.send(formData); + }); +} + +/** + * A 503 or transport failure reads as `pending`: the worker fleet is + * unreachable, which says nothing about this file. + * + * A failed task's `result` is the raw exception, which has no business on a + * third-party page, so no reason is returned and the caller supplies one. + */ +export async function fetchTaskOutcome( + taskId: string, + apiHost: string, + signal?: AbortSignal, +): Promise { + const response = await fetch( + `${apiHost}/api/task_status?task_id=${encodeURIComponent(taskId)}`, + { signal }, + ); + if (!response.ok) return { state: 'pending' }; + + const data = await response.json().catch(() => null); + if (data?.status === 'SUCCESS') { + const attachmentId = data?.result?.attachment_id; + return { + state: 'completed', + attachmentId: attachmentId ? String(attachmentId) : undefined, + }; + } + if (data?.status === 'FAILURE' || data?.status === 'REVOKED') + return { state: 'failed' }; + return { state: 'pending' }; +} diff --git a/extensions/react-widget/src/requests/streamingApi.ts b/extensions/react-widget/src/requests/streamingApi.ts index 9f8c1389..a5333417 100644 --- a/extensions/react-widget/src/requests/streamingApi.ts +++ b/extensions/react-widget/src/requests/streamingApi.ts @@ -10,6 +10,8 @@ interface FetchAnswerStreamingProps { history?: HistoryItem[]; conversationId?: string | null; apiHost?: string; + /** Server-side ids of attachments to send with this question. */ + attachments?: string[]; onEvent?: (event: MessageEvent) => void; signal?: AbortSignal; } @@ -29,13 +31,14 @@ export function fetchAnswerStreaming({ history = [], conversationId = null, apiHost = '', + attachments = [], onEvent = () => { console.log('Event triggered, but no handler provided.'); }, signal, }: FetchAnswerStreamingProps): Promise { return new Promise((resolve, reject) => { - const body = { + const body: Record = { question: question, history: JSON.stringify( history @@ -45,6 +48,7 @@ export function fetchAnswerStreaming({ conversation_id: conversationId, api_key: apiKey, }; + if (attachments.length > 0) body.attachments = attachments; fetch(apiHost + '/stream', { method: 'POST', headers: { diff --git a/extensions/react-widget/src/types/index.ts b/extensions/react-widget/src/types/index.ts index cfdb2c45..a87371dc 100644 --- a/extensions/react-widget/src/types/index.ts +++ b/extensions/react-widget/src/types/index.ts @@ -50,6 +50,30 @@ export type Status = 'idle' | 'loading' | 'failed'; export type FEEDBACK = 'LIKE' | 'DISLIKE'; +export type AttachmentStatus = + | 'uploading' + | 'processing' + | 'completed' + | 'failed'; + +export interface Attachment { + /** Client-side key for the chip; never sent to the server. */ + id: string; + fileName: string; + status: AttachmentStatus; + /** Only meaningful while `status` is 'uploading'. */ + progress: number; + /** Server-side id; what `/stream` expects in its `attachments` array. */ + attachmentId?: string; + /** Why it failed, when there is something worth showing. */ + error?: string; +} + +export interface SentAttachment { + id: string; + fileName: string; +} + export type THEME = 'light' | 'dark'; export interface Query { @@ -66,6 +90,8 @@ export interface Query { notice?: string; /** Tool names from tool_calls / tool_call. */ toolCalls?: string[]; + /** Files sent with the question. */ + attachments?: SentAttachment[]; } export interface WidgetProps { @@ -95,6 +121,21 @@ export interface WidgetProps { collectFeedback?: boolean; showSources?: boolean; defaultOpen?: boolean; + /** + * File extensions the composer accepts, e.g. `['.pdf', '.md', '.png']`. + * Attachments stay off until this is set, since every uploaded file is + * parsed and billed against the key owner's token budget. A leading dot is + * optional and matching is case-insensitive. + */ + allowedFileExtensions?: string[]; + /** + * Show the microphone that dictates into the input via the browser's Web + * Speech API. Off by default: outside Chromium builds with on-device + * recognition the browser forwards audio to its vendor's speech service. + * The button hides itself where the API is missing or the origin is + * insecure. + */ + showMicButton?: boolean; } export interface WidgetCoreProps extends WidgetProps { widgetRef?: React.RefObject | null;