Files
viettranx 04a9938f4f feat(tts): wire tenant timeout + fix Gemini text-only 400
- HTTP synthesize + test-connection now read tenant tts.timeout_ms
  (default 120s, was hardcoded 15s/10s). Gemini client default also
  bumped 30s→120s so both layers align when tenant config unset.
- Inline prefix "Speak naturally: " prepended to single-voice text;
  multi-speaker transcripts pass through unchanged.
- ErrTextOnlyResponse sentinel for 400 "text generation" bodies;
  single-voice retries once with stronger prefix. Narrowed needle
  list avoids false positives on unrelated 400s.
- SynthesizeWithFallbackAdapted now returns errors.Join so sentinel
  survives fallback chain; HTTP 422 mapping + locale-translated
  ForLLM in agent tool (EN/VI/ZH catalogs).
- Default Gemini model bumped to gemini-3.1-flash-tts-preview.
2026-04-23 08:31:53 +07:00

352 lines
12 KiB
Go

package audio
import (
"context"
"errors"
"fmt"
"log/slog"
"maps"
)
// ctxKeyChannel is the context key for the current channel name (e.g. "telegram").
// Set via WithChannel; read by resolveSTTChain to select channel-scoped providers.
type ctxKeyChannelType struct{}
var ctxKeyChannel = ctxKeyChannelType{}
// WithChannel returns a context that carries the channel name for STT chain resolution.
func WithChannel(ctx context.Context, channel string) context.Context {
return context.WithValue(ctx, ctxKeyChannel, channel)
}
// channelFromCtx extracts the channel name from ctx, or "" if not set.
func channelFromCtx(ctx context.Context) string {
v, _ := ctx.Value(ctxKeyChannel).(string)
return v
}
// TenantTTSResolver resolves per-tenant TTS provider and config.
// Returns (provider, providerName, autoMode, error). If error is non-nil,
// caller should fall back to global manager config.
type TenantTTSResolver func(ctx context.Context) (TTSProvider, string, AutoMode, error)
// Manager orchestrates audio providers across TTS, STT, Music, and SFX
// operations. Each op has its own provider map + primary/fallback chain.
//
// Phase 1 exercises the TTS path end-to-end. STT/Music/SFX maps and chains
// are present but empty (providers register in Phase 3/4).
type Manager struct {
ttsProviders map[string]TTSProvider
sttProviders map[string]STTProvider
musicProviders map[string]MusicProvider
sfxProviders map[string]SFXProvider
primary string // primary TTS provider
sttChain []string // STT fallback order (Phase 4)
musicChain []string // Music fallback order (Phase 3)
channelSTTOverrides map[string][]string // channel → provider key list (Phase 4)
auto AutoMode
mode Mode
maxLength int // max text length before truncation (default 1500)
timeoutMs int // provider timeout (default 30000)
tenantResolver TenantTTSResolver // per-tenant TTS config resolver (nil = use global)
}
// ManagerConfig configures the audio manager. Preserved from legacy TTS
// package — new STT/Music fields are set via RegisterSTT/RegisterMusic and
// (optionally) cfg.Audio in config_audio.go.
type ManagerConfig struct {
Primary string // primary TTS provider name
Auto AutoMode // auto-apply mode (default "off")
Mode Mode // "final" or "all" (default "final")
MaxLength int // default 1500
TimeoutMs int // default 30000
}
// NewManager creates an audio manager with empty provider maps.
func NewManager(cfg ManagerConfig) *Manager {
m := &Manager{
ttsProviders: make(map[string]TTSProvider),
sttProviders: make(map[string]STTProvider),
musicProviders: make(map[string]MusicProvider),
sfxProviders: make(map[string]SFXProvider),
channelSTTOverrides: make(map[string][]string),
primary: cfg.Primary,
auto: cfg.Auto,
mode: cfg.Mode,
maxLength: cfg.MaxLength,
timeoutMs: cfg.TimeoutMs,
}
if m.auto == "" {
m.auto = AutoOff
}
if m.mode == "" {
m.mode = ModeFinal
}
if m.maxLength <= 0 {
m.maxLength = 1500
}
if m.timeoutMs <= 0 {
m.timeoutMs = 30000
}
return m
}
// ---- Registration ----
// RegisterTTS adds a TTS provider. If no primary is set, the first registered
// provider becomes primary — matches legacy tts.Manager.RegisterProvider.
func (m *Manager) RegisterTTS(p TTSProvider) {
m.ttsProviders[p.Name()] = p
if m.primary == "" {
m.primary = p.Name()
}
}
// RegisterProvider is a backward-compat alias for RegisterTTS — lets pre-Phase-1
// callers that go through tts.Manager (= audio.Manager via alias) keep working.
func (m *Manager) RegisterProvider(p TTSProvider) { m.RegisterTTS(p) }
// RegisterSTT adds an STT provider (Phase 4).
func (m *Manager) RegisterSTT(p STTProvider) {
m.sttProviders[p.Name()] = p
}
// RegisterMusic adds a music provider (Phase 3).
func (m *Manager) RegisterMusic(p MusicProvider) {
m.musicProviders[p.Name()] = p
}
// RegisterSFX adds an SFX provider (Phase 3).
func (m *Manager) RegisterSFX(p SFXProvider) {
m.sfxProviders[p.Name()] = p
}
// ---- Introspection ----
// GetProvider returns a TTS provider by name. Preserved from legacy API.
func (m *Manager) GetProvider(name string) (TTSProvider, bool) {
p, ok := m.ttsProviders[name]
return p, ok
}
// PrimaryProvider returns the primary TTS provider name.
func (m *Manager) PrimaryProvider() string { return m.primary }
// SetTenantResolver sets the per-tenant TTS config resolver.
// Channels use this to resolve tenant-specific TTS providers.
func (m *Manager) SetTenantResolver(r TenantTTSResolver) { m.tenantResolver = r }
// ResolveTenantProvider attempts to get tenant-specific TTS provider.
// Returns (provider, name, autoMode, ok). If ok=false, caller uses global config.
func (m *Manager) ResolveTenantProvider(ctx context.Context) (TTSProvider, string, AutoMode, bool) {
if m.tenantResolver == nil {
return nil, "", "", false
}
p, name, auto, err := m.tenantResolver(ctx)
if err != nil {
return nil, "", "", false
}
return p, name, auto, true
}
// AutoMode returns the current auto-apply mode.
func (m *Manager) AutoMode() AutoMode { return m.auto }
// HasProviders reports whether any TTS provider is registered.
func (m *Manager) HasProviders() bool { return len(m.ttsProviders) > 0 }
// ListCapabilities iterates all registered TTS providers and returns their
// capability schemas. Providers implementing DescribableProvider contribute
// their full schema; others contribute a minimal stub {Provider, DisplayName}.
func (m *Manager) ListCapabilities() []ProviderCapabilities {
out := make([]ProviderCapabilities, 0, len(m.ttsProviders))
for _, p := range m.ttsProviders {
if dp, ok := p.(DescribableProvider); ok {
out = append(out, dp.Capabilities())
} else {
out = append(out, ProviderCapabilities{
Provider: p.Name(),
DisplayName: p.Name(),
})
}
}
return out
}
// ---- TTS dispatch ----
// Synthesize uses the primary provider.
func (m *Manager) Synthesize(ctx context.Context, text string, opts TTSOptions) (*SynthResult, error) {
p, ok := m.ttsProviders[m.primary]
if !ok {
return nil, fmt.Errorf("tts provider not found: %s", m.primary)
}
return p.Synthesize(ctx, text, opts)
}
// SynthesizeStream dispatches streaming TTS to the primary provider. Returns
// ErrStreamingNotSupported if the primary does not implement
// StreamingTTSProvider, letting callers fall back to buffered Synthesize.
func (m *Manager) SynthesizeStream(ctx context.Context, text string, opts TTSOptions) (*StreamResult, error) {
p, ok := m.ttsProviders[m.primary]
if !ok {
return nil, fmt.Errorf("tts provider not found: %s", m.primary)
}
sp, ok := p.(StreamingTTSProvider)
if !ok {
return nil, ErrStreamingNotSupported
}
return sp.SynthesizeStream(ctx, text, opts)
}
// ---- Music dispatch ----
// GenerateMusic tries registered music providers in chain order until one succeeds.
// Chain order: elevenlabs first (if registered), then remaining providers.
// Override order by setting m.musicChain before the first call.
func (m *Manager) GenerateMusic(ctx context.Context, opts MusicOptions) (*AudioResult, error) {
chain := m.resolveMusicChain()
if len(chain) == 0 {
return nil, fmt.Errorf("no music providers registered")
}
var lastErr error
for _, name := range chain {
p, ok := m.musicProviders[name]
if !ok {
slog.Info("audio.music provider not registered, skipping", "provider", name)
continue
}
if res, err := p.GenerateMusic(ctx, opts); err == nil {
return res, nil
} else {
slog.Warn("audio.music provider failed", "provider", name, "error", err)
lastErr = err
}
}
if lastErr != nil {
return nil, fmt.Errorf("all music providers failed: %w", lastErr)
}
return nil, fmt.Errorf("no music providers registered")
}
// GenerateSFX tries SFX providers in order: elevenlabs first, then any other registered.
func (m *Manager) GenerateSFX(ctx context.Context, opts SFXOptions) (*AudioResult, error) {
order := m.resolveSFXChain()
if len(order) == 0 {
return nil, fmt.Errorf("no sfx providers registered")
}
var lastErr error
for _, name := range order {
p, ok := m.sfxProviders[name]
if !ok {
continue
}
if res, err := p.GenerateSFX(ctx, opts); err == nil {
return res, nil
} else {
slog.Warn("audio.sfx provider failed", "provider", name, "error", err)
lastErr = err
}
}
if lastErr != nil {
return nil, fmt.Errorf("all sfx providers failed: %w", lastErr)
}
return nil, fmt.Errorf("no sfx providers registered")
}
// resolveMusicChain returns the ordered provider names for music generation.
// If m.musicChain is set explicitly it is used as-is; otherwise elevenlabs is
// preferred and remaining providers follow in registration order.
func (m *Manager) resolveMusicChain() []string {
if len(m.musicChain) > 0 {
return m.musicChain
}
out := make([]string, 0, len(m.musicProviders))
if _, ok := m.musicProviders["elevenlabs"]; ok {
out = append(out, "elevenlabs")
}
for name := range m.musicProviders {
if name != "elevenlabs" {
out = append(out, name)
}
}
return out
}
// resolveSFXChain returns the ordered provider names for SFX generation.
func (m *Manager) resolveSFXChain() []string {
out := make([]string, 0, len(m.sfxProviders))
if _, ok := m.sfxProviders["elevenlabs"]; ok {
out = append(out, "elevenlabs")
}
for name := range m.sfxProviders {
if name != "elevenlabs" {
out = append(out, name)
}
}
return out
}
// SynthesizeWithFallback tries primary first, then any other registered
// provider on error. Returns first success or aggregate failure.
func (m *Manager) SynthesizeWithFallback(ctx context.Context, text string, opts TTSOptions) (*SynthResult, error) {
return m.SynthesizeWithFallbackAdapted(ctx, text, opts, nil)
}
// SynthesizeWithFallbackAdapted is like SynthesizeWithFallback but applies
// AdaptAgentParams(genericAgentParams, providerName) per-attempt before
// synthesizing. This is the Finding #1 fix: each fallback attempt receives
// provider-native params rather than the primary's adapted keys.
//
// genericAgentParams must use the generic allow-list keys (speed, emotion, style).
// Passing nil is safe and produces the same behaviour as SynthesizeWithFallback.
func (m *Manager) SynthesizeWithFallbackAdapted(ctx context.Context, text string, opts TTSOptions, genericAgentParams map[string]any) (*SynthResult, error) {
var providerErrs []error
if p, ok := m.ttsProviders[m.primary]; ok {
attemptOpts := m.withAdaptedParams(opts, m.primary, genericAgentParams)
if result, err := p.Synthesize(ctx, text, attemptOpts); err == nil {
return result, nil
} else {
slog.Warn("tts primary provider failed, trying fallback", "provider", m.primary, "error", err)
providerErrs = append(providerErrs, fmt.Errorf("%s: %w", m.primary, err))
}
}
for name, p := range m.ttsProviders {
if name == m.primary {
continue
}
attemptOpts := m.withAdaptedParams(opts, name, genericAgentParams)
result, err := p.Synthesize(ctx, text, attemptOpts)
if err == nil {
slog.Info("tts fallback succeeded", "provider", name)
return result, nil
}
slog.Warn("tts fallback provider failed", "provider", name, "error", err)
providerErrs = append(providerErrs, fmt.Errorf("%s: %w", name, err))
}
if len(providerErrs) == 0 {
return nil, fmt.Errorf("no tts providers registered")
}
// errors.Join preserves all sentinel errors so errors.Is(err, sentinel) works downstream.
return nil, errors.Join(providerErrs...)
}
// withAdaptedParams returns a copy of opts with genericAgentParams adapted
// for providerName merged in. If genericAgentParams is nil or adaptation
// produces no keys, opts is returned unchanged.
func (m *Manager) withAdaptedParams(opts TTSOptions, providerName string, genericAgentParams map[string]any) TTSOptions {
adapted := AdaptAgentParams(genericAgentParams, providerName)
if len(adapted) == 0 {
return opts
}
out := opts
merged := make(map[string]any, len(opts.Params)+len(adapted))
maps.Copy(merged, opts.Params)
maps.Copy(merged, adapted)
out.Params = merged
return out
}