mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-04 16:13:47 +00:00
An agent's tts_voice_id (e.g. "en-US-GuyNeural" for Edge) was applied regardless of the selected TTS provider, causing OpenAI and other providers to reject unknown voice IDs. - Add internal/audio/voice_compat.go with IsVoiceCompatible, GetProviderDefaultVoice, and FilterVoiceForProvider helpers - Update TtsTool.resolveVoiceAndModel to report whether the voice came from an agent override (voiceFromAgent) - Apply applyVoiceCompat per-provider branch in Execute so incompatible agent voices fall back to the provider's default with a slog.Warn for operator visibility - Add voice_compat_test.go with table-driven tests for all cases Co-authored-by: Bruno Clermont <bruno.clermont@gmail.com>
384 lines
14 KiB
Go
384 lines
14 KiB
Go
package tools
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"maps"
|
|
"os"
|
|
"path/filepath"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
|
|
"github.com/nextlevelbuilder/goclaw/internal/audio"
|
|
"github.com/nextlevelbuilder/goclaw/internal/audio/gemini"
|
|
"github.com/nextlevelbuilder/goclaw/internal/bus"
|
|
"github.com/nextlevelbuilder/goclaw/internal/i18n"
|
|
"github.com/nextlevelbuilder/goclaw/internal/store"
|
|
"github.com/nextlevelbuilder/goclaw/internal/tts"
|
|
)
|
|
|
|
// TtsTool is an agent tool that converts text to speech audio.
|
|
// Matching TS src/agents/tools/tts-tool.ts.
|
|
// Implements Tool + ContextualTool interfaces.
|
|
// Per-call channel is read from ctx for thread-safety.
|
|
type TtsTool struct {
|
|
mu sync.RWMutex
|
|
manager *tts.Manager
|
|
vaultIntc *VaultInterceptor
|
|
}
|
|
|
|
func (t *TtsTool) SetVaultInterceptor(v *VaultInterceptor) { t.vaultIntc = v }
|
|
|
|
// NewTtsTool creates a TTS tool backed by the given manager.
|
|
func NewTtsTool(mgr *tts.Manager) *TtsTool {
|
|
return &TtsTool{manager: mgr}
|
|
}
|
|
|
|
// UpdateManager swaps the underlying TTS manager (used on config reload).
|
|
func (t *TtsTool) UpdateManager(mgr *tts.Manager) {
|
|
t.mu.Lock()
|
|
defer t.mu.Unlock()
|
|
t.manager = mgr
|
|
}
|
|
|
|
func (t *TtsTool) Name() string { return "tts" }
|
|
|
|
func (t *TtsTool) Description() string {
|
|
return "Convert text to speech audio. Returns a MEDIA: path to the generated audio file."
|
|
}
|
|
|
|
func (t *TtsTool) Parameters() map[string]any {
|
|
return map[string]any{
|
|
"type": "object",
|
|
"properties": map[string]any{
|
|
"text": map[string]any{
|
|
"type": "string",
|
|
"description": "The text to convert to speech",
|
|
},
|
|
"voice": map[string]any{
|
|
"type": "string",
|
|
"description": "Voice ID (provider-specific). Optional — uses default if omitted.",
|
|
},
|
|
"model": map[string]any{
|
|
"type": "string",
|
|
"description": "Model ID (provider-specific, e.g. eleven_v3). Optional — uses default if omitted.",
|
|
},
|
|
"provider": map[string]any{
|
|
"type": "string",
|
|
"description": "TTS provider: openai, elevenlabs, edge, minimax, gemini. Optional — uses configured primary if omitted.",
|
|
},
|
|
},
|
|
"required": []string{"text"},
|
|
}
|
|
}
|
|
|
|
// ttsOverride is the tenant settings shape for tts
|
|
// (stored in builtin_tool_tenant_configs.settings).
|
|
type ttsOverride struct {
|
|
Primary string `json:"primary,omitempty"` // override primary provider name
|
|
DefaultVoiceID string `json:"default_voice_id,omitempty"` // tenant-default voice id
|
|
DefaultModel string `json:"default_model,omitempty"` // tenant-default model id
|
|
}
|
|
|
|
// agentAudioConfig is the JSON shape read from AgentAudioSnapshot.OtherConfig
|
|
// for per-agent TTS tuning. Keys match the agents.other_config column.
|
|
type agentAudioConfig struct {
|
|
TTSVoiceID string `json:"tts_voice_id,omitempty"`
|
|
TTSModelID string `json:"tts_model_id,omitempty"`
|
|
// TTSParams carries the per-agent generic TTS override keys (speed, emotion, style).
|
|
// Stored as generic keys; AdaptAgentParams converts to provider-specific keys per attempt.
|
|
TTSParams map[string]any `json:"tts_params,omitempty"`
|
|
}
|
|
|
|
// resolveVoiceAndModel computes the effective voice + model IDs for the
|
|
// request using the documented precedence order:
|
|
//
|
|
// args > agent (store.AgentAudioFromCtx OtherConfig) > tenant (BuiltinToolSettings) > empty.
|
|
//
|
|
// Empty return values signal "use provider default" downstream — they are not
|
|
// errors. Missing agent snapshot emits slog.Warn so operators can spot
|
|
// dispatch-layer regressions; missing tenant settings are quiet (common).
|
|
//
|
|
// voiceFromAgent is true when the returned voice originated from the agent's
|
|
// tts_voice_id override (not from tool args or tenant defaults). Callers use
|
|
// this flag to emit a warning when the voice is later found incompatible with
|
|
// the selected provider.
|
|
func (t *TtsTool) resolveVoiceAndModel(ctx context.Context, argVoice, argModel string) (voice, model string, voiceFromAgent bool) {
|
|
voice, model = argVoice, argModel
|
|
|
|
// Pull agent-level config from the dispatcher-injected snapshot.
|
|
var agentCfg agentAudioConfig
|
|
if snap, ok := store.AgentAudioFromCtx(ctx); ok {
|
|
if len(snap.OtherConfig) > 0 {
|
|
if err := json.Unmarshal(snap.OtherConfig, &agentCfg); err != nil {
|
|
slog.Warn("tts: failed to parse agent OtherConfig", "error", err, "agent_id", snap.AgentID)
|
|
}
|
|
}
|
|
} else if agentID := store.AgentIDFromContext(ctx); agentID != uuid.Nil {
|
|
// Producer-consumer contract violation: when an agent ctx is in play
|
|
// (AgentIDFromContext returns non-nil), AgentAudioSnapshot should have
|
|
// been injected by the dispatcher. Log as Warn so ops can spot a
|
|
// dispatch-layer regression. Silent when no agent is scoped (unit
|
|
// tests and callers outside the agent loop).
|
|
slog.Warn("tts: agent audio snapshot missing — dispatcher producer may be offline", "agent_id", agentID)
|
|
}
|
|
|
|
// Pull tenant defaults from builtin tool settings.
|
|
var tenantCfg ttsOverride
|
|
if settings := BuiltinToolSettingsFromCtx(ctx); settings != nil {
|
|
if raw, ok := settings["tts"]; ok && len(raw) > 0 {
|
|
if err := json.Unmarshal(raw, &tenantCfg); err != nil {
|
|
slog.Warn("tts: failed to parse tenant settings for voice/model", "error", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
if voice == "" {
|
|
if agentCfg.TTSVoiceID != "" {
|
|
voice = agentCfg.TTSVoiceID
|
|
voiceFromAgent = true
|
|
} else if tenantCfg.DefaultVoiceID != "" {
|
|
voice = tenantCfg.DefaultVoiceID
|
|
}
|
|
}
|
|
if model == "" {
|
|
if agentCfg.TTSModelID != "" {
|
|
model = agentCfg.TTSModelID
|
|
} else if tenantCfg.DefaultModel != "" {
|
|
model = tenantCfg.DefaultModel
|
|
}
|
|
}
|
|
return voice, model, voiceFromAgent
|
|
}
|
|
|
|
// applyVoiceCompat checks whether voice is compatible with the named provider.
|
|
// When incompatible and the voice came from an agent override, it logs a
|
|
// warning and replaces the voice with the provider's default. The filtered
|
|
// voice (possibly unchanged) is returned.
|
|
func applyVoiceCompat(provider, voice string, voiceFromAgent bool) string {
|
|
filtered, changed := audio.FilterVoiceForProvider(provider, voice, voiceFromAgent)
|
|
if changed && voiceFromAgent {
|
|
slog.Warn("tts: agent tts_voice_id is incompatible with selected provider, using provider default",
|
|
"provider", provider,
|
|
"agent_voice", voice,
|
|
"fallback_voice", filtered,
|
|
)
|
|
}
|
|
return filtered
|
|
}
|
|
|
|
// resolvePrimary returns the effective primary provider name for the request.
|
|
// Checks tenant override via BuiltinToolSettingsFromCtx first.
|
|
func (t *TtsTool) resolvePrimary(ctx context.Context, mgr *tts.Manager) string {
|
|
if settings := BuiltinToolSettingsFromCtx(ctx); settings != nil {
|
|
if raw, ok := settings["tts"]; ok && len(raw) > 0 {
|
|
var override ttsOverride
|
|
if err := json.Unmarshal(raw, &override); err != nil {
|
|
slog.Warn("tts: failed to parse tenant override, using defaults", "error", err)
|
|
} else if override.Primary != "" {
|
|
// Verify the provider exists in the manager
|
|
if _, exists := mgr.GetProvider(override.Primary); exists {
|
|
return override.Primary
|
|
}
|
|
slog.Warn("tts: tenant override references unknown provider", "primary", override.Primary)
|
|
}
|
|
}
|
|
}
|
|
return mgr.PrimaryProvider()
|
|
}
|
|
|
|
// resolveTenantProvider returns the DB-configured TTS provider for this request.
|
|
// When requestedProvider is non-empty, only the matching tenant provider is
|
|
// accepted so an explicit provider never silently routes elsewhere.
|
|
func (t *TtsTool) resolveTenantProvider(ctx context.Context, mgr *tts.Manager, requestedProvider string) (audio.TTSProvider, string, bool) {
|
|
if mgr == nil {
|
|
return nil, "", false
|
|
}
|
|
p, name, _, ok := mgr.ResolveTenantProvider(ctx)
|
|
if !ok || p == nil || name == "" {
|
|
return nil, "", false
|
|
}
|
|
if requestedProvider != "" && requestedProvider != name {
|
|
return nil, "", false
|
|
}
|
|
return p, name, true
|
|
}
|
|
|
|
// resolveAgentGenericTTSParams reads the per-agent TTSParams generic map from
|
|
// the dispatcher-injected AgentAudioSnapshot. Returns nil when no snapshot
|
|
// is present or no tts_params are configured. The caller is responsible for
|
|
// calling audio.AdaptAgentParams(generic, providerName) PER-ATTEMPT to convert
|
|
// generic keys to provider-specific keys (Finding #1 CRITICAL).
|
|
func (t *TtsTool) resolveAgentGenericTTSParams(ctx context.Context) map[string]any {
|
|
snap, ok := store.AgentAudioFromCtx(ctx)
|
|
if !ok || len(snap.OtherConfig) == 0 {
|
|
return nil
|
|
}
|
|
var agentCfg agentAudioConfig
|
|
if err := json.Unmarshal(snap.OtherConfig, &agentCfg); err != nil {
|
|
slog.Warn("tts: failed to parse agent OtherConfig for tts_params", "error", err)
|
|
return nil
|
|
}
|
|
return agentCfg.TTSParams
|
|
}
|
|
|
|
// SetContext is a no-op; channel is now read from ctx (thread-safe).
|
|
func (t *TtsTool) SetContext(channel, _ string) {}
|
|
|
|
// mergeParams returns a new map with base keys overwritten by overrides.
|
|
// Returns overrides directly when base is nil (avoids allocation).
|
|
func mergeParams(base, overrides map[string]any) map[string]any {
|
|
if len(base) == 0 {
|
|
return overrides
|
|
}
|
|
out := make(map[string]any, len(base)+len(overrides))
|
|
maps.Copy(out, base)
|
|
maps.Copy(out, overrides)
|
|
return out
|
|
}
|
|
|
|
func (t *TtsTool) Execute(ctx context.Context, args map[string]any) *Result {
|
|
text, _ := args["text"].(string)
|
|
if text == "" {
|
|
return &Result{ForLLM: "error: text is required", IsError: true}
|
|
}
|
|
|
|
argVoice, _ := args["voice"].(string)
|
|
argModel, _ := args["model"].(string)
|
|
providerName, _ := args["provider"].(string)
|
|
|
|
// Resolve voice/model via args > agent (ctx snapshot) > tenant > default.
|
|
voice, model, voiceFromAgent := t.resolveVoiceAndModel(ctx, argVoice, argModel)
|
|
|
|
// Read generic agent TTS params once; adapt PER-ATTEMPT below (Finding #1 CRITICAL).
|
|
// Storing generic keys here so each fallback provider gets its own adapted copy.
|
|
genericAgentParams := t.resolveAgentGenericTTSParams(ctx)
|
|
|
|
// Snapshot manager pointer under read lock so config reloads don't race.
|
|
t.mu.RLock()
|
|
mgr := t.manager
|
|
t.mu.RUnlock()
|
|
|
|
// Determine format based on channel (read from ctx — thread-safe)
|
|
channel := ToolChannelFromCtx(ctx)
|
|
opts := tts.Options{Voice: voice, Model: model}
|
|
if channel == "telegram" {
|
|
opts.Format = "opus"
|
|
}
|
|
|
|
var result *tts.SynthResult
|
|
var err error
|
|
|
|
if providerName != "" {
|
|
// Use specific provider (explicit call param takes precedence).
|
|
// Adapt generic agent params to this specific provider's native keys.
|
|
p, ok := mgr.GetProvider(providerName)
|
|
if !ok {
|
|
var tenantName string
|
|
p, tenantName, ok = t.resolveTenantProvider(ctx, mgr, providerName)
|
|
if !ok {
|
|
return &Result{ForLLM: fmt.Sprintf("error: tts provider not found: %s", providerName), IsError: true}
|
|
}
|
|
providerName = tenantName
|
|
}
|
|
opts.Voice = applyVoiceCompat(providerName, opts.Voice, voiceFromAgent)
|
|
if adapted := audio.AdaptAgentParams(genericAgentParams, providerName); len(adapted) > 0 {
|
|
opts.Params = mergeParams(opts.Params, adapted)
|
|
}
|
|
result, err = p.Synthesize(ctx, text, opts)
|
|
} else {
|
|
// Prefer the DB-configured tenant provider used by /tts and auto-TTS.
|
|
if p, tenantName, ok := t.resolveTenantProvider(ctx, mgr, ""); ok {
|
|
tenantOpts := opts
|
|
tenantOpts.Voice = applyVoiceCompat(tenantName, tenantOpts.Voice, voiceFromAgent)
|
|
if adapted := audio.AdaptAgentParams(genericAgentParams, tenantName); len(adapted) > 0 {
|
|
tenantOpts.Params = mergeParams(opts.Params, adapted)
|
|
}
|
|
result, err = p.Synthesize(ctx, text, tenantOpts)
|
|
if err != nil {
|
|
slog.Warn("tts tenant provider failed, trying fallback", "provider", tenantName, "error", err)
|
|
result, err = mgr.SynthesizeWithFallbackAdapted(ctx, text, opts, genericAgentParams)
|
|
}
|
|
} else {
|
|
// Resolve primary from tenant settings or default.
|
|
primary := t.resolvePrimary(ctx, mgr)
|
|
if p, ok := mgr.GetProvider(primary); ok {
|
|
// Adapt for the primary provider attempt specifically.
|
|
primaryOpts := opts
|
|
primaryOpts.Voice = applyVoiceCompat(primary, primaryOpts.Voice, voiceFromAgent)
|
|
if adapted := audio.AdaptAgentParams(genericAgentParams, primary); len(adapted) > 0 {
|
|
primaryOpts.Params = mergeParams(opts.Params, adapted)
|
|
}
|
|
result, err = p.Synthesize(ctx, text, primaryOpts)
|
|
if err != nil {
|
|
slog.Warn("tts primary provider failed, trying fallback", "provider", primary, "error", err)
|
|
// SynthesizeWithFallbackAdapted adapts genericAgentParams per-attempt
|
|
// (Finding #1 CRITICAL): each fallback provider receives its own
|
|
// provider-native keys, not the primary's adapted map.
|
|
result, err = mgr.SynthesizeWithFallbackAdapted(ctx, text, opts, genericAgentParams)
|
|
}
|
|
} else {
|
|
result, err = mgr.SynthesizeWithFallbackAdapted(ctx, text, opts, genericAgentParams)
|
|
}
|
|
}
|
|
}
|
|
|
|
if err != nil {
|
|
if errors.Is(err, gemini.ErrTextOnlyResponse) {
|
|
locale := store.LocaleFromContext(ctx)
|
|
msg := i18n.T(locale, i18n.MsgTtsGeminiTextOnly)
|
|
return &Result{ForLLM: "error: " + msg, IsError: true}
|
|
}
|
|
return &Result{ForLLM: fmt.Sprintf("error: tts failed: %s", err.Error()), IsError: true}
|
|
}
|
|
|
|
// Write audio to workspace/tts/ so the agent can access the file.
|
|
// Falls back to os.TempDir() if workspace is not available.
|
|
ttsDir := os.TempDir()
|
|
if ws := ToolWorkspaceFromCtx(ctx); ws != "" {
|
|
ttsDir = filepath.Join(ws, "tts")
|
|
}
|
|
if err := os.MkdirAll(ttsDir, 0755); err != nil {
|
|
return &Result{ForLLM: fmt.Sprintf("error: create tts directory: %s", err.Error()), IsError: true}
|
|
}
|
|
audioPath := filepath.Join(ttsDir, fmt.Sprintf("tts-%d.%s", time.Now().UnixNano(), result.Extension))
|
|
if err := os.WriteFile(audioPath, result.Audio, 0644); err != nil {
|
|
return &Result{ForLLM: fmt.Sprintf("error: write tts audio: %s", err.Error()), IsError: true}
|
|
}
|
|
|
|
// Return MEDIA: path (matching TS pattern)
|
|
voiceTag := ""
|
|
if channel == "telegram" && result.Extension == "ogg" {
|
|
voiceTag = "[[audio_as_voice]]\n"
|
|
}
|
|
|
|
forLLM := fmt.Sprintf("%sMEDIA:%s", voiceTag, audioPath)
|
|
// Set Result.Media explicitly (matching create_audio) so the agent loop's
|
|
// media collector uses the authoritative path even when the ForLLM
|
|
// MEDIA: prefix is reshaped by a provider bridge (e.g. claude_cli MCP).
|
|
// Prefer the provider-supplied MimeType ("audio/mpeg", "audio/ogg") over
|
|
// "audio/"+Extension — the latter yields the non-standard "audio/mp3".
|
|
mimeType := result.MimeType
|
|
if mimeType == "" {
|
|
mimeType = "audio/" + result.Extension
|
|
}
|
|
r := &Result{
|
|
ForLLM: forLLM,
|
|
Media: []bus.MediaFile{{
|
|
Path: audioPath,
|
|
MimeType: mimeType,
|
|
}},
|
|
}
|
|
r.Deliverable = fmt.Sprintf("[Generated audio: %s]\nText: %s", filepath.Base(audioPath), text)
|
|
if t.vaultIntc != nil {
|
|
mimeType := "audio/" + result.Extension
|
|
go t.vaultIntc.AfterWriteMedia(context.WithoutCancel(ctx), audioPath, text, mimeType)
|
|
}
|
|
return r
|
|
}
|