Files
viettranx 247344d689 feat(tts): per-agent params override (speed/emotion/style) with adapter
Phase 4 — final phase of the TTS params/layout/agent-override plan.

Adds a 3-key allow-list (`speed`, `emotion`, `style`) per agent stored
in `agents.other_config.tts_params`. Backend resolves and merges into
`opts.Params` PER ATTEMPT inside the fallback loop so each provider
sees its own native shape — never the primary's keys when fallback
runs (Finding #1 critical).

Backend:
- `AgentOverridable bool` on `audio.ParamSchema`. UI filter reads this
  flag from /v1/tts/capabilities; no separate TS literal mirror —
  capabilities API is the single source of truth (Finding #9).
- `audio.AdaptAgentParams(generic, provider)` maps the 3 generic keys
  to provider-native paths (e.g. `speed` → `voice_settings.speed` for
  ElevenLabs, flat `speed` for OpenAI/MiniMax, dropped for Edge/Gemini).
- `Manager.SynthesizeWithFallbackAdapted` adapts inside the loop so
  fallback providers receive correctly-shaped params.
- `manager_auto.go` and `tools/tts.go` Execute do per-attempt adaptation
  on the tenant + direct + fallback call sites.
- Drop log bumped to `slog.Info("tts.agent.params.dropped", ...)` for
  audit trail when a generic key isn't supported by the active provider.
- Cross-check test asserts every adapter switch case has at least one
  capability ParamSchema with `AgentOverridable: true`, and vice versa.

Security (red-team findings):
- Allow-list ENFORCED at write path: `validateAgentTTSParams` in HTTP
  `handleUpdate` AND WS `agents_update` rejects any `tts_params` key
  outside `{speed, emotion, style}` (Finding #5).
- 64KB body cap on agent PUT via `http.MaxBytesReader` (Finding #6).
- Explicit tenant-scope guard after `agents.GetByID` (Finding #12).
- Concurrent-tab clobber: handleSave merges `tts_params` into a fresh
  copy of `otherConfig` rather than reusing stale state (Finding #13).
- Rate-limit verified — RoleAdmin gate sufficient for v1 (Finding #15).

Frontend (web + desktop):
- `TtsOverrideBlock` rewritten: filters capability params to
  `agent_overridable === true`, renders via `DynamicParamForm`. Hides
  entirely for providers with no overridable params (Edge, Gemini).
- Bidirectional adapter (generic ↔ capability-native form state) so
  agent storage stays in generic keys while UI works in native paths.
  25 round-trip tests cover all 5 providers.
- Desktop `AgentDetailPanel` gains an inline fine-tune section gated
  on `globalProvider`, reusing the desktop `DynamicParamForm`.

i18n: `tts.override.params.title` ("Fine-tune") added to web + desktop
en/vi/zh.

Tests: all 9 backend suites green (race), web 214/214, desktop build
clean, both Go build tags pass.
2026-04-20 06:46:19 +07:00

123 lines
3.6 KiB
Go

package audio
import (
"context"
"encoding/json"
"log/slog"
"maps"
"strings"
"github.com/nextlevelbuilder/goclaw/internal/store"
)
// MaybeApply inspects auto-mode and conditionally applies TTS to a reply.
// Returns (result, true) on success, (nil, false) when auto is disabled, the
// reply type is filtered out, content fails validation, or synthesis fails.
//
// Parameters:
// - text: the reply text to potentially convert
// - channel: origin channel ("telegram" switches format to opus)
// - isVoiceInbound: whether the user's inbound message was voice
// - kind: "tool", "block", or "final"
func (m *Manager) MaybeApply(ctx context.Context, text, channel string, isVoiceInbound bool, kind string) (*SynthResult, bool) {
// Try tenant-specific TTS config first
tenantProvider, _, tenantAuto, hasTenant := m.ResolveTenantProvider(ctx)
auto := m.auto
if hasTenant && tenantAuto != "" {
auto = tenantAuto
}
if auto == AutoOff {
return nil, false
}
// Mode filter: ModeFinal skips tool/block replies.
if m.mode == ModeFinal && (kind == "tool" || kind == "block") {
return nil, false
}
switch auto {
case AutoInbound:
if !isVoiceInbound {
return nil, false
}
case AutoTagged:
if !strings.Contains(text, "[[tts]]") && !strings.Contains(text, "[[tts:") {
return nil, false
}
case AutoAlways:
// Always apply.
default:
return nil, false
}
// Content validation (matches legacy TTS behavior).
cleanText := stripMarkdown(text)
cleanText = StripTTSDirectives(cleanText)
cleanText = strings.TrimSpace(cleanText)
if len(cleanText) < 10 {
return nil, false
}
if strings.Contains(cleanText, "MEDIA:") {
return nil, false
}
if len(cleanText) > m.maxLength {
cleanText = cleanText[:m.maxLength] + "..."
}
opts := TTSOptions{}
if channel == "telegram" {
opts.Format = "opus" // Telegram voice bubbles need opus
}
// Apply per-agent voice/model override from context (set by dispatch.go from OutboundMessage)
var agentGenericTTSParams map[string]any
if snap, ok := store.AgentAudioFromCtx(ctx); ok && len(snap.OtherConfig) > 0 {
var agentCfg struct {
TTSVoiceID string `json:"tts_voice_id,omitempty"`
TTSModelID string `json:"tts_model_id,omitempty"`
// TTSParams carries per-agent generic override keys (speed, emotion, style).
// Must be adapted PER-ATTEMPT via AdaptAgentParams (Finding #1 CRITICAL).
TTSParams map[string]any `json:"tts_params,omitempty"`
}
if err := json.Unmarshal(snap.OtherConfig, &agentCfg); err == nil {
if agentCfg.TTSVoiceID != "" {
opts.Voice = agentCfg.TTSVoiceID
}
if agentCfg.TTSModelID != "" {
opts.Model = agentCfg.TTSModelID
}
agentGenericTTSParams = agentCfg.TTSParams
}
}
var result *SynthResult
var err error
// Use tenant provider if available, otherwise fall back to global.
// Params are adapted PER-ATTEMPT so each provider receives its own native keys
// (Finding #1 CRITICAL: do NOT adapt once before the branch, adapt inside each path).
if hasTenant && tenantProvider != nil {
tenantOpts := opts
if adapted := AdaptAgentParams(agentGenericTTSParams, tenantProvider.Name()); len(adapted) > 0 {
merged := make(map[string]any, len(opts.Params)+len(adapted))
maps.Copy(merged, opts.Params)
maps.Copy(merged, adapted)
tenantOpts.Params = merged
}
result, err = tenantProvider.Synthesize(ctx, cleanText, tenantOpts)
} else {
// SynthesizeWithFallbackAdapted adapts per-attempt inside the fallback loop.
result, err = m.SynthesizeWithFallbackAdapted(ctx, cleanText, opts, agentGenericTTSParams)
}
if err != nil {
slog.Warn("tts auto-apply failed", "error", err)
return nil, false
}
return result, true
}