Files
goclaw/internal/tools/tts.go
T
Bruno ClermontandBruno Clermont 0184678dc3 fix(tts): validate voice compatibility with selected provider before synthesis (#1302)
An agent's tts_voice_id (e.g. "en-US-GuyNeural" for Edge) was applied
regardless of the selected TTS provider, causing OpenAI and other
providers to reject unknown voice IDs.

- Add internal/audio/voice_compat.go with IsVoiceCompatible,
  GetProviderDefaultVoice, and FilterVoiceForProvider helpers
- Update TtsTool.resolveVoiceAndModel to report whether the voice
  came from an agent override (voiceFromAgent)
- Apply applyVoiceCompat per-provider branch in Execute so
  incompatible agent voices fall back to the provider's default
  with a slog.Warn for operator visibility
- Add voice_compat_test.go with table-driven tests for all cases

Co-authored-by: Bruno Clermont <bruno.clermont@gmail.com>
2026-06-29 06:54:45 +07:00

384 lines
14 KiB
Go

package tools
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"maps"
"os"
"path/filepath"
"sync"
"time"
"github.com/google/uuid"
"github.com/nextlevelbuilder/goclaw/internal/audio"
"github.com/nextlevelbuilder/goclaw/internal/audio/gemini"
"github.com/nextlevelbuilder/goclaw/internal/bus"
"github.com/nextlevelbuilder/goclaw/internal/i18n"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/tts"
)
// TtsTool is an agent tool that converts text to speech audio.
// Matching TS src/agents/tools/tts-tool.ts.
// Implements Tool + ContextualTool interfaces.
// Per-call channel is read from ctx for thread-safety.
type TtsTool struct {
mu sync.RWMutex
manager *tts.Manager
vaultIntc *VaultInterceptor
}
func (t *TtsTool) SetVaultInterceptor(v *VaultInterceptor) { t.vaultIntc = v }
// NewTtsTool creates a TTS tool backed by the given manager.
func NewTtsTool(mgr *tts.Manager) *TtsTool {
return &TtsTool{manager: mgr}
}
// UpdateManager swaps the underlying TTS manager (used on config reload).
func (t *TtsTool) UpdateManager(mgr *tts.Manager) {
t.mu.Lock()
defer t.mu.Unlock()
t.manager = mgr
}
func (t *TtsTool) Name() string { return "tts" }
func (t *TtsTool) Description() string {
return "Convert text to speech audio. Returns a MEDIA: path to the generated audio file."
}
func (t *TtsTool) Parameters() map[string]any {
return map[string]any{
"type": "object",
"properties": map[string]any{
"text": map[string]any{
"type": "string",
"description": "The text to convert to speech",
},
"voice": map[string]any{
"type": "string",
"description": "Voice ID (provider-specific). Optional — uses default if omitted.",
},
"model": map[string]any{
"type": "string",
"description": "Model ID (provider-specific, e.g. eleven_v3). Optional — uses default if omitted.",
},
"provider": map[string]any{
"type": "string",
"description": "TTS provider: openai, elevenlabs, edge, minimax, gemini. Optional — uses configured primary if omitted.",
},
},
"required": []string{"text"},
}
}
// ttsOverride is the tenant settings shape for tts
// (stored in builtin_tool_tenant_configs.settings).
type ttsOverride struct {
Primary string `json:"primary,omitempty"` // override primary provider name
DefaultVoiceID string `json:"default_voice_id,omitempty"` // tenant-default voice id
DefaultModel string `json:"default_model,omitempty"` // tenant-default model id
}
// agentAudioConfig is the JSON shape read from AgentAudioSnapshot.OtherConfig
// for per-agent TTS tuning. Keys match the agents.other_config column.
type agentAudioConfig struct {
TTSVoiceID string `json:"tts_voice_id,omitempty"`
TTSModelID string `json:"tts_model_id,omitempty"`
// TTSParams carries the per-agent generic TTS override keys (speed, emotion, style).
// Stored as generic keys; AdaptAgentParams converts to provider-specific keys per attempt.
TTSParams map[string]any `json:"tts_params,omitempty"`
}
// resolveVoiceAndModel computes the effective voice + model IDs for the
// request using the documented precedence order:
//
// args > agent (store.AgentAudioFromCtx OtherConfig) > tenant (BuiltinToolSettings) > empty.
//
// Empty return values signal "use provider default" downstream — they are not
// errors. Missing agent snapshot emits slog.Warn so operators can spot
// dispatch-layer regressions; missing tenant settings are quiet (common).
//
// voiceFromAgent is true when the returned voice originated from the agent's
// tts_voice_id override (not from tool args or tenant defaults). Callers use
// this flag to emit a warning when the voice is later found incompatible with
// the selected provider.
func (t *TtsTool) resolveVoiceAndModel(ctx context.Context, argVoice, argModel string) (voice, model string, voiceFromAgent bool) {
voice, model = argVoice, argModel
// Pull agent-level config from the dispatcher-injected snapshot.
var agentCfg agentAudioConfig
if snap, ok := store.AgentAudioFromCtx(ctx); ok {
if len(snap.OtherConfig) > 0 {
if err := json.Unmarshal(snap.OtherConfig, &agentCfg); err != nil {
slog.Warn("tts: failed to parse agent OtherConfig", "error", err, "agent_id", snap.AgentID)
}
}
} else if agentID := store.AgentIDFromContext(ctx); agentID != uuid.Nil {
// Producer-consumer contract violation: when an agent ctx is in play
// (AgentIDFromContext returns non-nil), AgentAudioSnapshot should have
// been injected by the dispatcher. Log as Warn so ops can spot a
// dispatch-layer regression. Silent when no agent is scoped (unit
// tests and callers outside the agent loop).
slog.Warn("tts: agent audio snapshot missing — dispatcher producer may be offline", "agent_id", agentID)
}
// Pull tenant defaults from builtin tool settings.
var tenantCfg ttsOverride
if settings := BuiltinToolSettingsFromCtx(ctx); settings != nil {
if raw, ok := settings["tts"]; ok && len(raw) > 0 {
if err := json.Unmarshal(raw, &tenantCfg); err != nil {
slog.Warn("tts: failed to parse tenant settings for voice/model", "error", err)
}
}
}
if voice == "" {
if agentCfg.TTSVoiceID != "" {
voice = agentCfg.TTSVoiceID
voiceFromAgent = true
} else if tenantCfg.DefaultVoiceID != "" {
voice = tenantCfg.DefaultVoiceID
}
}
if model == "" {
if agentCfg.TTSModelID != "" {
model = agentCfg.TTSModelID
} else if tenantCfg.DefaultModel != "" {
model = tenantCfg.DefaultModel
}
}
return voice, model, voiceFromAgent
}
// applyVoiceCompat checks whether voice is compatible with the named provider.
// When incompatible and the voice came from an agent override, it logs a
// warning and replaces the voice with the provider's default. The filtered
// voice (possibly unchanged) is returned.
func applyVoiceCompat(provider, voice string, voiceFromAgent bool) string {
filtered, changed := audio.FilterVoiceForProvider(provider, voice, voiceFromAgent)
if changed && voiceFromAgent {
slog.Warn("tts: agent tts_voice_id is incompatible with selected provider, using provider default",
"provider", provider,
"agent_voice", voice,
"fallback_voice", filtered,
)
}
return filtered
}
// resolvePrimary returns the effective primary provider name for the request.
// Checks tenant override via BuiltinToolSettingsFromCtx first.
func (t *TtsTool) resolvePrimary(ctx context.Context, mgr *tts.Manager) string {
if settings := BuiltinToolSettingsFromCtx(ctx); settings != nil {
if raw, ok := settings["tts"]; ok && len(raw) > 0 {
var override ttsOverride
if err := json.Unmarshal(raw, &override); err != nil {
slog.Warn("tts: failed to parse tenant override, using defaults", "error", err)
} else if override.Primary != "" {
// Verify the provider exists in the manager
if _, exists := mgr.GetProvider(override.Primary); exists {
return override.Primary
}
slog.Warn("tts: tenant override references unknown provider", "primary", override.Primary)
}
}
}
return mgr.PrimaryProvider()
}
// resolveTenantProvider returns the DB-configured TTS provider for this request.
// When requestedProvider is non-empty, only the matching tenant provider is
// accepted so an explicit provider never silently routes elsewhere.
func (t *TtsTool) resolveTenantProvider(ctx context.Context, mgr *tts.Manager, requestedProvider string) (audio.TTSProvider, string, bool) {
if mgr == nil {
return nil, "", false
}
p, name, _, ok := mgr.ResolveTenantProvider(ctx)
if !ok || p == nil || name == "" {
return nil, "", false
}
if requestedProvider != "" && requestedProvider != name {
return nil, "", false
}
return p, name, true
}
// resolveAgentGenericTTSParams reads the per-agent TTSParams generic map from
// the dispatcher-injected AgentAudioSnapshot. Returns nil when no snapshot
// is present or no tts_params are configured. The caller is responsible for
// calling audio.AdaptAgentParams(generic, providerName) PER-ATTEMPT to convert
// generic keys to provider-specific keys (Finding #1 CRITICAL).
func (t *TtsTool) resolveAgentGenericTTSParams(ctx context.Context) map[string]any {
snap, ok := store.AgentAudioFromCtx(ctx)
if !ok || len(snap.OtherConfig) == 0 {
return nil
}
var agentCfg agentAudioConfig
if err := json.Unmarshal(snap.OtherConfig, &agentCfg); err != nil {
slog.Warn("tts: failed to parse agent OtherConfig for tts_params", "error", err)
return nil
}
return agentCfg.TTSParams
}
// SetContext is a no-op; channel is now read from ctx (thread-safe).
func (t *TtsTool) SetContext(channel, _ string) {}
// mergeParams returns a new map with base keys overwritten by overrides.
// Returns overrides directly when base is nil (avoids allocation).
func mergeParams(base, overrides map[string]any) map[string]any {
if len(base) == 0 {
return overrides
}
out := make(map[string]any, len(base)+len(overrides))
maps.Copy(out, base)
maps.Copy(out, overrides)
return out
}
func (t *TtsTool) Execute(ctx context.Context, args map[string]any) *Result {
text, _ := args["text"].(string)
if text == "" {
return &Result{ForLLM: "error: text is required", IsError: true}
}
argVoice, _ := args["voice"].(string)
argModel, _ := args["model"].(string)
providerName, _ := args["provider"].(string)
// Resolve voice/model via args > agent (ctx snapshot) > tenant > default.
voice, model, voiceFromAgent := t.resolveVoiceAndModel(ctx, argVoice, argModel)
// Read generic agent TTS params once; adapt PER-ATTEMPT below (Finding #1 CRITICAL).
// Storing generic keys here so each fallback provider gets its own adapted copy.
genericAgentParams := t.resolveAgentGenericTTSParams(ctx)
// Snapshot manager pointer under read lock so config reloads don't race.
t.mu.RLock()
mgr := t.manager
t.mu.RUnlock()
// Determine format based on channel (read from ctx — thread-safe)
channel := ToolChannelFromCtx(ctx)
opts := tts.Options{Voice: voice, Model: model}
if channel == "telegram" {
opts.Format = "opus"
}
var result *tts.SynthResult
var err error
if providerName != "" {
// Use specific provider (explicit call param takes precedence).
// Adapt generic agent params to this specific provider's native keys.
p, ok := mgr.GetProvider(providerName)
if !ok {
var tenantName string
p, tenantName, ok = t.resolveTenantProvider(ctx, mgr, providerName)
if !ok {
return &Result{ForLLM: fmt.Sprintf("error: tts provider not found: %s", providerName), IsError: true}
}
providerName = tenantName
}
opts.Voice = applyVoiceCompat(providerName, opts.Voice, voiceFromAgent)
if adapted := audio.AdaptAgentParams(genericAgentParams, providerName); len(adapted) > 0 {
opts.Params = mergeParams(opts.Params, adapted)
}
result, err = p.Synthesize(ctx, text, opts)
} else {
// Prefer the DB-configured tenant provider used by /tts and auto-TTS.
if p, tenantName, ok := t.resolveTenantProvider(ctx, mgr, ""); ok {
tenantOpts := opts
tenantOpts.Voice = applyVoiceCompat(tenantName, tenantOpts.Voice, voiceFromAgent)
if adapted := audio.AdaptAgentParams(genericAgentParams, tenantName); len(adapted) > 0 {
tenantOpts.Params = mergeParams(opts.Params, adapted)
}
result, err = p.Synthesize(ctx, text, tenantOpts)
if err != nil {
slog.Warn("tts tenant provider failed, trying fallback", "provider", tenantName, "error", err)
result, err = mgr.SynthesizeWithFallbackAdapted(ctx, text, opts, genericAgentParams)
}
} else {
// Resolve primary from tenant settings or default.
primary := t.resolvePrimary(ctx, mgr)
if p, ok := mgr.GetProvider(primary); ok {
// Adapt for the primary provider attempt specifically.
primaryOpts := opts
primaryOpts.Voice = applyVoiceCompat(primary, primaryOpts.Voice, voiceFromAgent)
if adapted := audio.AdaptAgentParams(genericAgentParams, primary); len(adapted) > 0 {
primaryOpts.Params = mergeParams(opts.Params, adapted)
}
result, err = p.Synthesize(ctx, text, primaryOpts)
if err != nil {
slog.Warn("tts primary provider failed, trying fallback", "provider", primary, "error", err)
// SynthesizeWithFallbackAdapted adapts genericAgentParams per-attempt
// (Finding #1 CRITICAL): each fallback provider receives its own
// provider-native keys, not the primary's adapted map.
result, err = mgr.SynthesizeWithFallbackAdapted(ctx, text, opts, genericAgentParams)
}
} else {
result, err = mgr.SynthesizeWithFallbackAdapted(ctx, text, opts, genericAgentParams)
}
}
}
if err != nil {
if errors.Is(err, gemini.ErrTextOnlyResponse) {
locale := store.LocaleFromContext(ctx)
msg := i18n.T(locale, i18n.MsgTtsGeminiTextOnly)
return &Result{ForLLM: "error: " + msg, IsError: true}
}
return &Result{ForLLM: fmt.Sprintf("error: tts failed: %s", err.Error()), IsError: true}
}
// Write audio to workspace/tts/ so the agent can access the file.
// Falls back to os.TempDir() if workspace is not available.
ttsDir := os.TempDir()
if ws := ToolWorkspaceFromCtx(ctx); ws != "" {
ttsDir = filepath.Join(ws, "tts")
}
if err := os.MkdirAll(ttsDir, 0755); err != nil {
return &Result{ForLLM: fmt.Sprintf("error: create tts directory: %s", err.Error()), IsError: true}
}
audioPath := filepath.Join(ttsDir, fmt.Sprintf("tts-%d.%s", time.Now().UnixNano(), result.Extension))
if err := os.WriteFile(audioPath, result.Audio, 0644); err != nil {
return &Result{ForLLM: fmt.Sprintf("error: write tts audio: %s", err.Error()), IsError: true}
}
// Return MEDIA: path (matching TS pattern)
voiceTag := ""
if channel == "telegram" && result.Extension == "ogg" {
voiceTag = "[[audio_as_voice]]\n"
}
forLLM := fmt.Sprintf("%sMEDIA:%s", voiceTag, audioPath)
// Set Result.Media explicitly (matching create_audio) so the agent loop's
// media collector uses the authoritative path even when the ForLLM
// MEDIA: prefix is reshaped by a provider bridge (e.g. claude_cli MCP).
// Prefer the provider-supplied MimeType ("audio/mpeg", "audio/ogg") over
// "audio/"+Extension — the latter yields the non-standard "audio/mp3".
mimeType := result.MimeType
if mimeType == "" {
mimeType = "audio/" + result.Extension
}
r := &Result{
ForLLM: forLLM,
Media: []bus.MediaFile{{
Path: audioPath,
MimeType: mimeType,
}},
}
r.Deliverable = fmt.Sprintf("[Generated audio: %s]\nText: %s", filepath.Base(audioPath), text)
if t.vaultIntc != nil {
mimeType := "audio/" + result.Extension
go t.vaultIntc.AfterWriteMedia(context.WithoutCancel(ctx), audioPath, text, mimeType)
}
return r
}