From e033b357cc34a7db36f53e63f35be4d6d567904f Mon Sep 17 00:00:00 2001 From: viettranx Date: Sat, 18 Apr 2026 14:54:51 +0700 Subject: [PATCH] fix(tools): route read_audio transcription models to /audio/transcriptions (#858) Transcription-only models (whisper-1, gpt-4o-transcribe, gpt-4o-mini-transcribe) were sent to /chat/completions with an input_audio content part, which OpenAI rejects with "not a chat model". Add isTranscriptionModel detection and a multipart/form-data call to /v1/audio/transcriptions for those models; chat-audio models (gpt-4o-audio-preview) continue to use /chat/completions. Also fix providerTypeFromName to treat "openai-*" prefixed providers as "openai" type so a dedicated "openai-audio" provider reaches the OpenAI-specific path instead of falling through to the generic chat fallback. --- internal/tools/media_provider_chain.go | 2 +- internal/tools/openai_transcription_call.go | 111 ++++++++++++++++++++ internal/tools/read_audio_resolve.go | 11 +- 3 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 internal/tools/openai_transcription_call.go diff --git a/internal/tools/media_provider_chain.go b/internal/tools/media_provider_chain.go index 53583db5..e62ab897 100644 --- a/internal/tools/media_provider_chain.go +++ b/internal/tools/media_provider_chain.go @@ -340,7 +340,7 @@ func providerTypeFromName(name string) string { return "minimax" case name == "alibaba" || name == "dashscope" || name == "bailian": return "dashscope" - case name == "openai": + case name == "openai" || strings.HasPrefix(name, "openai-"): return "openai" case name == "anthropic": return "anthropic" diff --git a/internal/tools/openai_transcription_call.go b/internal/tools/openai_transcription_call.go new file mode 100644 index 00000000..58ff06ca --- /dev/null +++ b/internal/tools/openai_transcription_call.go @@ -0,0 +1,111 @@ +package tools + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "strings" + "time" + + "github.com/nextlevelbuilder/goclaw/internal/providers" +) + +// isTranscriptionModel returns true for OpenAI models that require the +// /v1/audio/transcriptions endpoint instead of /chat/completions. +// Covers whisper and the gpt-4o-(mini-)transcribe family. +func isTranscriptionModel(model string) bool { + m := strings.ToLower(strings.TrimSpace(model)) + if m == "" { + return false + } + if strings.HasPrefix(m, "whisper") { + return true + } + // gpt-4o-transcribe, gpt-4o-mini-transcribe, and future variants. + return strings.Contains(m, "transcribe") +} + +// extFromMime maps an audio MIME type to a file extension accepted by +// OpenAI's transcription endpoint. Falls back to .mp3 for unknown types. +func extFromMime(mime string) string { + m := strings.ToLower(mime) + switch { + case strings.Contains(m, "wav"): + return ".wav" + case strings.Contains(m, "mp3"), strings.Contains(m, "mpeg"): + return ".mp3" + case strings.Contains(m, "m4a"), strings.Contains(m, "mp4"): + return ".m4a" + case strings.Contains(m, "ogg"), strings.Contains(m, "opus"): + return ".ogg" + case strings.Contains(m, "flac"): + return ".flac" + case strings.Contains(m, "webm"): + return ".webm" + default: + return ".mp3" + } +} + +// openaiTranscriptionCall sends audio to OpenAI's /v1/audio/transcriptions +// endpoint using multipart/form-data. The endpoint returns only the +// transcribed text and does not provide token usage counters. +func openaiTranscriptionCall(ctx context.Context, apiKey, baseURL, model string, data []byte, mime string) (*providers.ChatResponse, error) { + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + + filePart, err := w.CreateFormFile("file", "audio"+extFromMime(mime)) + if err != nil { + return nil, fmt.Errorf("create form file: %w", err) + } + if _, err := filePart.Write(data); err != nil { + return nil, fmt.Errorf("write audio payload: %w", err) + } + if err := w.WriteField("model", model); err != nil { + return nil, fmt.Errorf("write model field: %w", err) + } + if err := w.Close(); err != nil { + return nil, fmt.Errorf("close multipart: %w", err) + } + + url := strings.TrimRight(baseURL, "/") + "/audio/transcriptions" + req, err := http.NewRequestWithContext(ctx, "POST", url, &buf) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + req.Header.Set("Content-Type", w.FormDataContentType()) + req.Header.Set("Authorization", "Bearer "+apiKey) + + client := &http.Client{Timeout: 120 * time.Second} + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("HTTP request: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, truncateStr(string(respBody), 500)) + } + + var out struct { + Text string `json:"text"` + } + if err := json.Unmarshal(respBody, &out); err != nil { + return nil, fmt.Errorf("parse response: %w", err) + } + if out.Text == "" { + return nil, fmt.Errorf("empty transcription") + } + return &providers.ChatResponse{ + Content: out.Text, + FinishReason: "stop", + }, nil +} diff --git a/internal/tools/read_audio_resolve.go b/internal/tools/read_audio_resolve.go index 2adc823a..e76baded 100644 --- a/internal/tools/read_audio_resolve.go +++ b/internal/tools/read_audio_resolve.go @@ -95,8 +95,17 @@ func (t *ReadAudioTool) callProvider(ctx context.Context, cp credentialProvider, return []byte(resp.Content), resp.Usage, nil } - // OpenAI: use input_audio content part (supports wav, mp3). + // OpenAI: transcription models need /v1/audio/transcriptions (multipart); + // chat-audio models use /chat/completions with input_audio content part. if ptype == "openai" { + if isTranscriptionModel(model) { + slog.Info("read_audio: using openai transcription API", "provider", providerName, "model", model, "size", len(data), "mime", mime) + resp, err := openaiTranscriptionCall(ctx, cp.APIKey(), cp.APIBase(), model, data, mime) + if err != nil { + return nil, nil, fmt.Errorf("openai transcription call: %w", err) + } + return []byte(resp.Content), resp.Usage, nil + } slog.Info("read_audio: using openai input_audio API", "provider", providerName, "model", model, "size", len(data), "mime", mime) resp, err := openaiAudioCall(ctx, cp.APIKey(), cp.APIBase(), model, prompt, data, mime) if err != nil {