mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-08-23 02:24:04 +00:00
Phase 4 — final phase of the TTS params/layout/agent-override plan. Adds a 3-key allow-list (`speed`, `emotion`, `style`) per agent stored in `agents.other_config.tts_params`. Backend resolves and merges into `opts.Params` PER ATTEMPT inside the fallback loop so each provider sees its own native shape — never the primary's keys when fallback runs (Finding #1 critical). Backend: - `AgentOverridable bool` on `audio.ParamSchema`. UI filter reads this flag from /v1/tts/capabilities; no separate TS literal mirror — capabilities API is the single source of truth (Finding #9). - `audio.AdaptAgentParams(generic, provider)` maps the 3 generic keys to provider-native paths (e.g. `speed` → `voice_settings.speed` for ElevenLabs, flat `speed` for OpenAI/MiniMax, dropped for Edge/Gemini). - `Manager.SynthesizeWithFallbackAdapted` adapts inside the loop so fallback providers receive correctly-shaped params. - `manager_auto.go` and `tools/tts.go` Execute do per-attempt adaptation on the tenant + direct + fallback call sites. - Drop log bumped to `slog.Info("tts.agent.params.dropped", ...)` for audit trail when a generic key isn't supported by the active provider. - Cross-check test asserts every adapter switch case has at least one capability ParamSchema with `AgentOverridable: true`, and vice versa. Security (red-team findings): - Allow-list ENFORCED at write path: `validateAgentTTSParams` in HTTP `handleUpdate` AND WS `agents_update` rejects any `tts_params` key outside `{speed, emotion, style}` (Finding #5). - 64KB body cap on agent PUT via `http.MaxBytesReader` (Finding #6). - Explicit tenant-scope guard after `agents.GetByID` (Finding #12). - Concurrent-tab clobber: handleSave merges `tts_params` into a fresh copy of `otherConfig` rather than reusing stale state (Finding #13). - Rate-limit verified — RoleAdmin gate sufficient for v1 (Finding #15). Frontend (web + desktop): - `TtsOverrideBlock` rewritten: filters capability params to `agent_overridable === true`, renders via `DynamicParamForm`. Hides entirely for providers with no overridable params (Edge, Gemini). - Bidirectional adapter (generic ↔ capability-native form state) so agent storage stays in generic keys while UI works in native paths. 25 round-trip tests cover all 5 providers. - Desktop `AgentDetailPanel` gains an inline fine-tune section gated on `globalProvider`, reusing the desktop `DynamicParamForm`. i18n: `tts.override.params.title` ("Fine-tune") added to web + desktop en/vi/zh. Tests: all 9 backend suites green (race), web 214/214, desktop build clean, both Go build tags pass.
123 lines
3.6 KiB
Go
123 lines
3.6 KiB
Go
package audio
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"log/slog"
|
|
"maps"
|
|
"strings"
|
|
|
|
"github.com/nextlevelbuilder/goclaw/internal/store"
|
|
)
|
|
|
|
// MaybeApply inspects auto-mode and conditionally applies TTS to a reply.
|
|
// Returns (result, true) on success, (nil, false) when auto is disabled, the
|
|
// reply type is filtered out, content fails validation, or synthesis fails.
|
|
//
|
|
// Parameters:
|
|
// - text: the reply text to potentially convert
|
|
// - channel: origin channel ("telegram" switches format to opus)
|
|
// - isVoiceInbound: whether the user's inbound message was voice
|
|
// - kind: "tool", "block", or "final"
|
|
func (m *Manager) MaybeApply(ctx context.Context, text, channel string, isVoiceInbound bool, kind string) (*SynthResult, bool) {
|
|
// Try tenant-specific TTS config first
|
|
tenantProvider, _, tenantAuto, hasTenant := m.ResolveTenantProvider(ctx)
|
|
|
|
auto := m.auto
|
|
if hasTenant && tenantAuto != "" {
|
|
auto = tenantAuto
|
|
}
|
|
|
|
if auto == AutoOff {
|
|
return nil, false
|
|
}
|
|
|
|
// Mode filter: ModeFinal skips tool/block replies.
|
|
if m.mode == ModeFinal && (kind == "tool" || kind == "block") {
|
|
return nil, false
|
|
}
|
|
|
|
switch auto {
|
|
case AutoInbound:
|
|
if !isVoiceInbound {
|
|
return nil, false
|
|
}
|
|
case AutoTagged:
|
|
if !strings.Contains(text, "[[tts]]") && !strings.Contains(text, "[[tts:") {
|
|
return nil, false
|
|
}
|
|
case AutoAlways:
|
|
// Always apply.
|
|
default:
|
|
return nil, false
|
|
}
|
|
|
|
// Content validation (matches legacy TTS behavior).
|
|
cleanText := stripMarkdown(text)
|
|
cleanText = StripTTSDirectives(cleanText)
|
|
cleanText = strings.TrimSpace(cleanText)
|
|
|
|
if len(cleanText) < 10 {
|
|
return nil, false
|
|
}
|
|
if strings.Contains(cleanText, "MEDIA:") {
|
|
return nil, false
|
|
}
|
|
|
|
if len(cleanText) > m.maxLength {
|
|
cleanText = cleanText[:m.maxLength] + "..."
|
|
}
|
|
|
|
opts := TTSOptions{}
|
|
if channel == "telegram" {
|
|
opts.Format = "opus" // Telegram voice bubbles need opus
|
|
}
|
|
|
|
// Apply per-agent voice/model override from context (set by dispatch.go from OutboundMessage)
|
|
var agentGenericTTSParams map[string]any
|
|
if snap, ok := store.AgentAudioFromCtx(ctx); ok && len(snap.OtherConfig) > 0 {
|
|
var agentCfg struct {
|
|
TTSVoiceID string `json:"tts_voice_id,omitempty"`
|
|
TTSModelID string `json:"tts_model_id,omitempty"`
|
|
// TTSParams carries per-agent generic override keys (speed, emotion, style).
|
|
// Must be adapted PER-ATTEMPT via AdaptAgentParams (Finding #1 CRITICAL).
|
|
TTSParams map[string]any `json:"tts_params,omitempty"`
|
|
}
|
|
if err := json.Unmarshal(snap.OtherConfig, &agentCfg); err == nil {
|
|
if agentCfg.TTSVoiceID != "" {
|
|
opts.Voice = agentCfg.TTSVoiceID
|
|
}
|
|
if agentCfg.TTSModelID != "" {
|
|
opts.Model = agentCfg.TTSModelID
|
|
}
|
|
agentGenericTTSParams = agentCfg.TTSParams
|
|
}
|
|
}
|
|
|
|
var result *SynthResult
|
|
var err error
|
|
|
|
// Use tenant provider if available, otherwise fall back to global.
|
|
// Params are adapted PER-ATTEMPT so each provider receives its own native keys
|
|
// (Finding #1 CRITICAL: do NOT adapt once before the branch, adapt inside each path).
|
|
if hasTenant && tenantProvider != nil {
|
|
tenantOpts := opts
|
|
if adapted := AdaptAgentParams(agentGenericTTSParams, tenantProvider.Name()); len(adapted) > 0 {
|
|
merged := make(map[string]any, len(opts.Params)+len(adapted))
|
|
maps.Copy(merged, opts.Params)
|
|
maps.Copy(merged, adapted)
|
|
tenantOpts.Params = merged
|
|
}
|
|
result, err = tenantProvider.Synthesize(ctx, cleanText, tenantOpts)
|
|
} else {
|
|
// SynthesizeWithFallbackAdapted adapts per-attempt inside the fallback loop.
|
|
result, err = m.SynthesizeWithFallbackAdapted(ctx, cleanText, opts, agentGenericTTSParams)
|
|
}
|
|
|
|
if err != nil {
|
|
slog.Warn("tts auto-apply failed", "error", err)
|
|
return nil, false
|
|
}
|
|
return result, true
|
|
}
|