diff --git a/cmd/gateway_builtin_tools.go b/cmd/gateway_builtin_tools.go index 2f5922b0..b0cd6d99 100644 --- a/cmd/gateway_builtin_tools.go +++ b/cmd/gateway_builtin_tools.go @@ -51,6 +51,18 @@ func builtinToolSeedData() []store.BuiltinToolDef { Settings: json.RawMessage(`{"provider":"openrouter","model":"google/gemini-2.5-flash-image"}`), Requires: []string{"image_gen_provider"}, }, + {Name: "read_audio", DisplayName: "Read Audio", Description: "Analyze audio files (speech, music, sounds) using an audio-capable LLM provider", Category: "media", Enabled: true, + Settings: json.RawMessage(`{"provider":"gemini","model":"gemini-2.5-flash"}`), + Requires: []string{"audio_provider"}, + }, + {Name: "read_video", DisplayName: "Read Video", Description: "Analyze video files using a video-capable LLM provider", Category: "media", Enabled: true, + Settings: json.RawMessage(`{"provider":"gemini","model":"gemini-2.5-flash"}`), + Requires: []string{"video_provider"}, + }, + {Name: "create_video", DisplayName: "Create Video", Description: "Generate videos from text descriptions using AI", Category: "media", Enabled: true, + Settings: json.RawMessage(`{"provider":"gemini","model":"veo-3.0-generate-preview"}`), + Requires: []string{"video_gen_provider"}, + }, {Name: "tts", DisplayName: "Text to Speech", Description: "Convert text to speech audio", Category: "media", Enabled: true, Requires: []string{"tts_provider"}, Metadata: json.RawMessage(`{"config_hint":"Config → TTS"}`), diff --git a/cmd/gateway_managed.go b/cmd/gateway_managed.go index 22e52172..660ac362 100644 --- a/cmd/gateway_managed.go +++ b/cmd/gateway_managed.go @@ -77,9 +77,12 @@ func wireExtras( _ = mediaStore.DeleteSession(sessionKey) } } - // Register document analysis tool (needs mediaStore for file access). + // Register media analysis tools (need mediaStore for file access). toolsReg.Register(tools.NewReadDocumentTool(providerReg, mediaStore)) - slog.Info("read_document tool registered") + toolsReg.Register(tools.NewReadAudioTool(providerReg, mediaStore)) + toolsReg.Register(tools.NewReadVideoTool(providerReg, mediaStore)) + toolsReg.Register(tools.NewCreateVideoTool(providerReg)) + slog.Info("media tools registered", "tools", "read_document,read_audio,read_video,create_video") } // 2. User seeding callback: seeds per-user context files on first chat diff --git a/internal/agent/loop.go b/internal/agent/loop.go index 77d1de32..9e9ade3b 100644 --- a/internal/agent/loop.go +++ b/internal/agent/loop.go @@ -359,7 +359,43 @@ func (l *Loop) runLoop(ctx context.Context, req RunRequest) (*RunResult, error) ctx = tools.WithMediaDocRefs(ctx, docRefs) } - // 2b. Cross-session recovery: notify team leads about orphaned pending tasks + // 2c. Collect audio MediaRefs (current + historical) for read_audio tool. + var audioRefs []providers.MediaRef + for _, ref := range mediaRefs { + if ref.Kind == "audio" { + audioRefs = append(audioRefs, ref) + } + } + for i := len(messages) - 1; i >= 0; i-- { + for _, ref := range messages[i].MediaRefs { + if ref.Kind == "audio" { + audioRefs = append(audioRefs, ref) + } + } + } + if len(audioRefs) > 0 { + ctx = tools.WithMediaAudioRefs(ctx, audioRefs) + } + + // 2d. Collect video MediaRefs (current + historical) for read_video tool. + var videoRefs []providers.MediaRef + for _, ref := range mediaRefs { + if ref.Kind == "video" { + videoRefs = append(videoRefs, ref) + } + } + for i := len(messages) - 1; i >= 0; i-- { + for _, ref := range messages[i].MediaRefs { + if ref.Kind == "video" { + videoRefs = append(videoRefs, ref) + } + } + } + if len(videoRefs) > 0 { + ctx = tools.WithMediaVideoRefs(ctx, videoRefs) + } + + // 2e. Cross-session recovery: notify team leads about orphaned pending tasks // and in-progress tasks being handled by delegates. // Safe because Bước 1 (early ClaimTask) ensures running tasks are in_progress, // so only truly un-spawned tasks remain pending. diff --git a/internal/agent/loop_history.go b/internal/agent/loop_history.go index 287cf894..3dcc35fa 100644 --- a/internal/agent/loop_history.go +++ b/internal/agent/loop_history.go @@ -15,6 +15,20 @@ import ( "github.com/nextlevelbuilder/goclaw/internal/store" ) +// filteredToolNames returns tool names after applying policy filters. +// Used for system prompt so denied tools don't appear in ## Tooling section. +func (l *Loop) filteredToolNames() []string { + if l.toolPolicy == nil { + return l.tools.List() + } + defs := l.toolPolicy.FilterTools(l.tools, l.id, l.provider.Name(), l.agentToolPolicy, nil, false, false) + names := make([]string, len(defs)) + for i, d := range defs { + names[i] = d.Function.Name + } + return names +} + // buildMessages constructs the full message list for an LLM request. // Returns the messages and whether BOOTSTRAP.md was present in context files // (used by the caller for auto-cleanup without an extra DB roundtrip). @@ -73,7 +87,7 @@ func (l *Loop) buildMessages(ctx context.Context, history []providers.Message, s PeerKind: peerKind, OwnerIDs: l.ownerIDs, Mode: mode, - ToolNames: l.tools.List(), + ToolNames: l.filteredToolNames(), SkillsSummary: l.resolveSkillsSummary(skillFilter), HasMemory: l.hasMemory, HasSpawn: l.tools != nil && hasSpawn, diff --git a/internal/agent/memoryflush.go b/internal/agent/memoryflush.go index 46d1af74..b8dbc249 100644 --- a/internal/agent/memoryflush.go +++ b/internal/agent/memoryflush.go @@ -128,7 +128,7 @@ func (l *Loop) runMemoryFlush(ctx context.Context, sessionKey string, settings * Model: l.model, Workspace: l.workspace, Mode: PromptMinimal, - ToolNames: l.tools.List(), + ToolNames: l.filteredToolNames(), HasMemory: l.hasMemory, }) systemPrompt += "\n\n" + settings.SystemPrompt diff --git a/internal/agent/systemprompt.go b/internal/agent/systemprompt.go index 1e9c7cd3..a74fbdc8 100644 --- a/internal/agent/systemprompt.go +++ b/internal/agent/systemprompt.go @@ -71,6 +71,9 @@ var coreToolSummaries = map[string]string{ "sessions_history": "Fetch message history for a session", "sessions_send": "Send a message into another session", "read_image": "Analyze images attached to the conversation. MUST call this when you see tags", + "read_audio": "Analyze audio files attached to the conversation. MUST call this when you see tags", + "read_video": "Analyze video files attached to the conversation. MUST call this when you see tags", + "create_video": "Generate videos from text descriptions using AI", "read_document": "Analyze documents (PDF, DOCX, etc.) attached to the conversation. MUST call this when you see tags", "create_image": "Generate images from text descriptions using AI", } diff --git a/internal/tools/create_video.go b/internal/tools/create_video.go new file mode 100644 index 00000000..a1b9a318 --- /dev/null +++ b/internal/tools/create_video.go @@ -0,0 +1,360 @@ +package tools + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/nextlevelbuilder/goclaw/internal/bus" + "github.com/nextlevelbuilder/goclaw/internal/providers" +) + +// videoGenProviderPriority is the default order for video generation providers. +var videoGenProviderPriority = []string{"gemini", "openrouter"} + +// videoGenModelDefaults maps provider names to default video generation models. +var videoGenModelDefaults = map[string]string{ + "gemini": "veo-3.0-generate-preview", + "openrouter": "google/veo-3.0-generate-preview", +} + +// CreateVideoTool generates videos using a video generation API. +// Uses Gemini Veo via native generateContent API with VIDEO response modality. +type CreateVideoTool struct { + registry *providers.Registry +} + +func NewCreateVideoTool(registry *providers.Registry) *CreateVideoTool { + return &CreateVideoTool{registry: registry} +} + +func (t *CreateVideoTool) Name() string { return "create_video" } + +func (t *CreateVideoTool) Description() string { + return "Generate a video from a text description using a video generation model. Returns a MEDIA: path to the generated video file." +} + +func (t *CreateVideoTool) Parameters() map[string]interface{} { + return map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "prompt": map[string]interface{}{ + "type": "string", + "description": "Text description of the video to generate.", + }, + "duration": map[string]interface{}{ + "type": "integer", + "description": "Video duration in seconds (default 5, max 30).", + }, + "aspect_ratio": map[string]interface{}{ + "type": "string", + "description": "Aspect ratio: '16:9' (default), '9:16', '1:1'.", + }, + }, + "required": []string{"prompt"}, + } +} + +func (t *CreateVideoTool) Execute(ctx context.Context, args map[string]interface{}) *Result { + prompt, _ := args["prompt"].(string) + if prompt == "" { + return ErrorResult("prompt is required") + } + + // Parse and enforce duration (default 5, max 30). + duration := 5 + if d, ok := args["duration"].(float64); ok { + duration = int(d) + } + if duration < 1 { + duration = 1 + } + if duration > 30 { + duration = 30 + } + + // Parse and validate aspect ratio. + aspectRatio := "16:9" + if ar, _ := args["aspect_ratio"].(string); ar != "" { + switch ar { + case "16:9", "9:16", "1:1": + aspectRatio = ar + default: + return ErrorResult(fmt.Sprintf("unsupported aspect_ratio %q; use '16:9', '9:16', or '1:1'", ar)) + } + } + + providerName, model := t.resolveConfig(ctx) + + p, err := t.registry.Get(providerName) + if err != nil { + return ErrorResult(fmt.Sprintf("video generation provider %q not available", providerName)) + } + + cp, ok := p.(credentialProvider) + if !ok { + return ErrorResult(fmt.Sprintf("provider %q does not expose API credentials for video generation", providerName)) + } + + slog.Info("create_video: calling video generation API", "provider", providerName, "model", model, "duration", duration, "aspect_ratio", aspectRatio) + + var videoBytes []byte + var usage *providers.Usage + switch providerName { + case "gemini": + videoBytes, usage, err = t.callGeminiVideoGen(ctx, cp.APIKey(), cp.APIBase(), model, prompt, duration, aspectRatio) + default: + // OpenRouter and others: try chat completions with VIDEO modality + videoBytes, usage, err = t.callChatVideoGen(ctx, cp.APIKey(), cp.APIBase(), model, prompt, duration, aspectRatio) + } + if err != nil { + return ErrorResult(fmt.Sprintf("video generation failed: %v", err)) + } + + // Save to workspace under date-based folder. + workspace := ToolWorkspaceFromCtx(ctx) + if workspace == "" { + workspace = os.TempDir() + } + dateDir := filepath.Join(workspace, "generated", time.Now().Format("2006-01-02")) + if err := os.MkdirAll(dateDir, 0755); err != nil { + return ErrorResult(fmt.Sprintf("failed to create output directory: %v", err)) + } + videoPath := filepath.Join(dateDir, fmt.Sprintf("goclaw_gen_%d.mp4", time.Now().UnixNano())) + if err := os.WriteFile(videoPath, videoBytes, 0644); err != nil { + return ErrorResult(fmt.Sprintf("failed to save generated video: %v", err)) + } + + result := &Result{ForLLM: fmt.Sprintf("MEDIA:%s", videoPath)} + result.Media = []bus.MediaFile{{Path: videoPath, MimeType: "video/mp4"}} + result.Deliverable = fmt.Sprintf("[Generated video: %s]\nPrompt: %s", filepath.Base(videoPath), prompt) + result.Provider = providerName + result.Model = model + if usage != nil { + result.Usage = usage + } + return result +} + +// resolveConfig returns the provider name and model for video generation. +func (t *CreateVideoTool) resolveConfig(ctx context.Context) (providerName, model string) { + // 1. Check global builtin_tools.settings + if settings := BuiltinToolSettingsFromCtx(ctx); settings != nil { + if raw, ok := settings["create_video"]; ok && len(raw) > 0 { + var cfg struct { + Provider string `json:"provider"` + Model string `json:"model"` + } + if json.Unmarshal(raw, &cfg) == nil && cfg.Provider != "" { + if _, err := t.registry.Get(cfg.Provider); err == nil { + providerName = cfg.Provider + model = cfg.Model + } + } + } + } + + // 2. Find first available from priority list + if providerName == "" { + for _, name := range videoGenProviderPriority { + if _, err := t.registry.Get(name); err == nil { + providerName = name + break + } + } + } + if providerName == "" { + providerName = "gemini" + } + + // 3. Default model + if model == "" { + if m, ok := videoGenModelDefaults[providerName]; ok { + model = m + } + } + + return providerName, model +} + +// callGeminiVideoGen uses the native Gemini generateContent API with VIDEO response modality. +func (t *CreateVideoTool) callGeminiVideoGen(ctx context.Context, apiKey, apiBase, model, prompt string, duration int, aspectRatio string) ([]byte, *providers.Usage, error) { + nativeBase := strings.TrimRight(apiBase, "/") + nativeBase = strings.TrimSuffix(nativeBase, "/openai") + + url := fmt.Sprintf("%s/models/%s:generateContent?key=%s", nativeBase, model, apiKey) + + genConfig := map[string]interface{}{ + "responseModalities": []string{"VIDEO"}, + "videoDuration": duration, + "aspectRatio": aspectRatio, + } + + body := map[string]interface{}{ + "contents": []map[string]interface{}{ + {"parts": []map[string]interface{}{{"text": prompt}}}, + }, + "generationConfig": genConfig, + } + + jsonBody, err := json.Marshal(body) + if err != nil { + return nil, nil, fmt.Errorf("marshal request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBody)) + if err != nil { + return nil, nil, fmt.Errorf("create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + // Video generation can take a while. + client := &http.Client{Timeout: 300 * time.Second} + resp, err := client.Do(req) + if err != nil { + return nil, nil, fmt.Errorf("http request: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, nil, fmt.Errorf("read response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, nil, fmt.Errorf("API error %d: %s", resp.StatusCode, truncateBytes(respBody, 500)) + } + + // Parse Gemini response — look for inlineData with video MIME. + var gemResp struct { + Candidates []struct { + Content struct { + Parts []struct { + InlineData *struct { + MimeType string `json:"mimeType"` + Data string `json:"data"` + } `json:"inlineData"` + } `json:"parts"` + } `json:"content"` + } `json:"candidates"` + UsageMetadata *struct { + PromptTokenCount int `json:"promptTokenCount"` + CandidatesTokenCount int `json:"candidatesTokenCount"` + TotalTokenCount int `json:"totalTokenCount"` + } `json:"usageMetadata"` + } + if err := json.Unmarshal(respBody, &gemResp); err != nil { + return nil, nil, fmt.Errorf("parse response: %w", err) + } + + for _, cand := range gemResp.Candidates { + for _, part := range cand.Content.Parts { + if part.InlineData != nil && strings.HasPrefix(part.InlineData.MimeType, "video/") { + videoBytes, err := base64.StdEncoding.DecodeString(part.InlineData.Data) + if err != nil { + return nil, nil, fmt.Errorf("decode base64: %w", err) + } + var usage *providers.Usage + if gemResp.UsageMetadata != nil { + usage = &providers.Usage{ + PromptTokens: gemResp.UsageMetadata.PromptTokenCount, + CompletionTokens: gemResp.UsageMetadata.CandidatesTokenCount, + TotalTokens: gemResp.UsageMetadata.TotalTokenCount, + } + } + return videoBytes, usage, nil + } + } + } + + return nil, nil, fmt.Errorf("no video data in Gemini response") +} + +// callChatVideoGen tries OpenAI-compatible chat completions with video modality. +func (t *CreateVideoTool) callChatVideoGen(ctx context.Context, apiKey, apiBase, model, prompt string, duration int, aspectRatio string) ([]byte, *providers.Usage, error) { + body := map[string]interface{}{ + "model": model, + "messages": []map[string]interface{}{ + {"role": "user", "content": prompt}, + }, + "modalities": []string{"video", "text"}, + "duration": duration, + "aspect_ratio": aspectRatio, + } + + jsonBody, err := json.Marshal(body) + if err != nil { + return nil, nil, fmt.Errorf("marshal request: %w", err) + } + + url := strings.TrimRight(apiBase, "/") + "/chat/completions" + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBody)) + if err != nil { + return nil, nil, fmt.Errorf("create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+apiKey) + + client := &http.Client{Timeout: 300 * time.Second} + resp, err := client.Do(req) + if err != nil { + return nil, nil, fmt.Errorf("http request: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, nil, fmt.Errorf("read response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, nil, fmt.Errorf("API error %d: %s", resp.StatusCode, truncateBytes(respBody, 500)) + } + + // Try to extract video from multipart content or data URL. + var chatResp struct { + Choices []struct { + Message struct { + Content interface{} `json:"content"` + } `json:"message"` + } `json:"choices"` + Usage *struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + } `json:"usage"` + } + if err := json.Unmarshal(respBody, &chatResp); err != nil { + return nil, nil, fmt.Errorf("parse response: %w", err) + } + + if len(chatResp.Choices) == 0 { + return nil, nil, fmt.Errorf("no choices in response") + } + + // Look for video data URL in multipart content. + if parts, ok := chatResp.Choices[0].Message.Content.([]interface{}); ok { + for _, part := range parts { + if m, ok := part.(map[string]interface{}); ok { + if m["type"] == "video_url" || m["type"] == "image_url" { + if vidURL, ok := m["video_url"].(map[string]interface{}); ok { + if urlStr, ok := vidURL["url"].(string); ok { + if videoBytes, err := decodeDataURL(urlStr); err == nil { + return videoBytes, convertUsage(chatResp.Usage), nil + } + } + } + } + } + } + } + + return nil, nil, fmt.Errorf("no video data found in response") +} diff --git a/internal/tools/gemini_file_api.go b/internal/tools/gemini_file_api.go new file mode 100644 index 00000000..82e5dcd3 --- /dev/null +++ b/internal/tools/gemini_file_api.go @@ -0,0 +1,261 @@ +package tools + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "time" + + "github.com/nextlevelbuilder/goclaw/internal/providers" +) + +const ( + geminiUploadBase = "https://generativelanguage.googleapis.com/upload/v1beta/files" + geminiFilesBase = "https://generativelanguage.googleapis.com/v1beta" + + geminiFilePollInterval = 2 * time.Second + geminiFilePollMax = 30 +) + +// geminiFileUpload uploads a file to Gemini File API using resumable upload protocol. +// Returns the file name (e.g. "files/abc123") and file URI for use in generateContent. +func geminiFileUpload(ctx context.Context, apiKey, displayName string, data []byte, mime string) (fileName, fileURI string, err error) { + // Step 1: Initiate resumable upload. + initBody, _ := json.Marshal(map[string]interface{}{ + "file": map[string]string{"display_name": displayName}, + }) + initReq, err := http.NewRequestWithContext(ctx, "POST", geminiUploadBase+"?key="+apiKey, bytes.NewReader(initBody)) + if err != nil { + return "", "", fmt.Errorf("create init request: %w", err) + } + initReq.Header.Set("Content-Type", "application/json") + initReq.Header.Set("X-Goog-Upload-Protocol", "resumable") + initReq.Header.Set("X-Goog-Upload-Command", "start") + initReq.Header.Set("X-Goog-Upload-Header-Content-Length", fmt.Sprintf("%d", len(data))) + initReq.Header.Set("X-Goog-Upload-Header-Content-Type", mime) + + client := &http.Client{Timeout: 60 * time.Second} + initResp, err := client.Do(initReq) + if err != nil { + return "", "", fmt.Errorf("init upload: %w", err) + } + defer initResp.Body.Close() + io.ReadAll(initResp.Body) // drain + + if initResp.StatusCode != 200 { + return "", "", fmt.Errorf("init upload HTTP %d", initResp.StatusCode) + } + + uploadURL := initResp.Header.Get("X-Goog-Upload-URL") + if uploadURL == "" { + return "", "", fmt.Errorf("no upload URL in response headers") + } + + // Step 2: Upload file bytes. + uploadReq, err := http.NewRequestWithContext(ctx, "POST", uploadURL, bytes.NewReader(data)) + if err != nil { + return "", "", fmt.Errorf("create upload request: %w", err) + } + uploadReq.Header.Set("Content-Length", fmt.Sprintf("%d", len(data))) + uploadReq.Header.Set("X-Goog-Upload-Offset", "0") + uploadReq.Header.Set("X-Goog-Upload-Command", "upload, finalize") + + uploadClient := &http.Client{Timeout: 120 * time.Second} + uploadResp, err := uploadClient.Do(uploadReq) + if err != nil { + return "", "", fmt.Errorf("upload bytes: %w", err) + } + defer uploadResp.Body.Close() + + respBody, err := io.ReadAll(uploadResp.Body) + if err != nil { + return "", "", fmt.Errorf("read upload response: %w", err) + } + if uploadResp.StatusCode != 200 { + return "", "", fmt.Errorf("upload HTTP %d: %s", uploadResp.StatusCode, truncateStr(string(respBody), 500)) + } + + var uploadResult struct { + File struct { + Name string `json:"name"` + URI string `json:"uri"` + } `json:"file"` + } + if err := json.Unmarshal(respBody, &uploadResult); err != nil { + return "", "", fmt.Errorf("parse upload response: %w", err) + } + + return uploadResult.File.Name, uploadResult.File.URI, nil +} + +// geminiFilePoll polls the Gemini File API until the file reaches ACTIVE state. +// Returns the file URI once active, or error on FAILED/timeout. +func geminiFilePoll(ctx context.Context, apiKey, fileName string) (fileURI string, err error) { + url := fmt.Sprintf("%s/%s?key=%s", geminiFilesBase, fileName, apiKey) + + for i := 0; i < geminiFilePollMax; i++ { + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(geminiFilePollInterval): + } + + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return "", fmt.Errorf("create poll request: %w", err) + } + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Do(req) + if err != nil { + slog.Warn("gemini file poll error, retrying", "attempt", i, "error", err) + continue + } + + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + + var status struct { + State string `json:"state"` + URI string `json:"uri"` + } + if err := json.Unmarshal(body, &status); err != nil { + slog.Warn("gemini file poll parse error, retrying", "attempt", i, "error", err) + continue + } + + switch status.State { + case "ACTIVE": + return status.URI, nil + case "FAILED": + return "", fmt.Errorf("file processing failed") + default: + slog.Debug("gemini file poll", "state", status.State, "attempt", i) + } + } + + return "", fmt.Errorf("file processing timeout after %d polls", geminiFilePollMax) +} + +// geminiFileAPICall uploads a file via Gemini File API, polls until ready, +// then calls generateContent with file_data reference. +// Used for audio/video files where inlineData doesn't work. +func geminiFileAPICall(ctx context.Context, apiKey, model, prompt string, data []byte, mime string, httpTimeout time.Duration) (*providers.ChatResponse, error) { + displayName := fmt.Sprintf("goclaw_%d", time.Now().UnixNano()) + + slog.Info("gemini file api: uploading", "size", len(data), "mime", mime) + fileName, fileURI, err := geminiFileUpload(ctx, apiKey, displayName, data, mime) + if err != nil { + return nil, fmt.Errorf("upload: %w", err) + } + slog.Info("gemini file api: uploaded", "name", fileName) + + // If file URI not returned directly, poll for it. + if fileURI == "" { + slog.Info("gemini file api: polling for active state", "name", fileName) + fileURI, err = geminiFilePoll(ctx, apiKey, fileName) + if err != nil { + return nil, fmt.Errorf("poll: %w", err) + } + } + slog.Info("gemini file api: file active", "uri", fileURI) + + // Call generateContent with file_data reference. + body := map[string]interface{}{ + "contents": []map[string]interface{}{ + { + "parts": []map[string]interface{}{ + {"file_data": map[string]interface{}{"mime_type": mime, "file_uri": fileURI}}, + {"text": prompt}, + }, + }, + }, + "generationConfig": map[string]interface{}{ + "maxOutputTokens": 16384, + "temperature": 0.2, + }, + } + + bodyJSON, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("marshal request: %w", err) + } + + url := fmt.Sprintf("https://generativelanguage.googleapis.com/v1beta/models/%s:generateContent?key=%s", model, apiKey) + httpReq, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyJSON)) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + httpReq.Header.Set("Content-Type", "application/json") + + if httpTimeout == 0 { + httpTimeout = 120 * time.Second + } + client := &http.Client{Timeout: httpTimeout} + httpResp, err := client.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("HTTP request: %w", err) + } + defer httpResp.Body.Close() + + respBody, err := io.ReadAll(httpResp.Body) + if err != nil { + return nil, fmt.Errorf("read response: %w", err) + } + if httpResp.StatusCode != 200 { + return nil, fmt.Errorf("HTTP %d: %s", httpResp.StatusCode, truncateStr(string(respBody), 500)) + } + + // Parse — same response format as geminiNativeDocumentCall. + return parseGeminiResponse(respBody) +} + +// parseGeminiResponse extracts text content and usage from a Gemini generateContent response. +func parseGeminiResponse(respBody []byte) (*providers.ChatResponse, error) { + var geminiResp struct { + Candidates []struct { + Content struct { + Parts []struct { + Text string `json:"text"` + } `json:"parts"` + } `json:"content"` + } `json:"candidates"` + UsageMetadata struct { + PromptTokenCount int `json:"promptTokenCount"` + CandidatesTokenCount int `json:"candidatesTokenCount"` + TotalTokenCount int `json:"totalTokenCount"` + } `json:"usageMetadata"` + } + if err := json.Unmarshal(respBody, &geminiResp); err != nil { + return nil, fmt.Errorf("parse response: %w", err) + } + + var content string + if len(geminiResp.Candidates) > 0 { + for _, part := range geminiResp.Candidates[0].Content.Parts { + if part.Text != "" { + if content != "" { + content += "\n" + } + content += part.Text + } + } + } + if content == "" { + return nil, fmt.Errorf("empty response from Gemini") + } + + return &providers.ChatResponse{ + Content: content, + FinishReason: "stop", + Usage: &providers.Usage{ + PromptTokens: geminiResp.UsageMetadata.PromptTokenCount, + CompletionTokens: geminiResp.UsageMetadata.CandidatesTokenCount, + TotalTokens: geminiResp.UsageMetadata.TotalTokenCount, + }, + }, nil +} diff --git a/internal/tools/openai_compat_call.go b/internal/tools/openai_compat_call.go new file mode 100644 index 00000000..1cd7dda2 --- /dev/null +++ b/internal/tools/openai_compat_call.go @@ -0,0 +1,80 @@ +package tools + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/nextlevelbuilder/goclaw/internal/providers" +) + +// callOpenAICompatJSON sends a raw JSON body to an OpenAI-compatible chat/completions endpoint +// and parses the response into a ChatResponse. Used by audio/video tools that need custom +// message formats not supported by the generic providers.ChatRequest interface. +func callOpenAICompatJSON(ctx context.Context, apiKey, baseURL string, body map[string]interface{}, timeout time.Duration) (*providers.ChatResponse, error) { + jsonBody, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("marshal request: %w", err) + } + + url := strings.TrimRight(baseURL, "/") + "/chat/completions" + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBody)) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+apiKey) + + client := &http.Client{Timeout: timeout} + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("HTTP request: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, truncateStr(string(respBody), 500)) + } + + // Parse standard OpenAI chat completion response. + var oaiResp struct { + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` + Usage *struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + } `json:"usage"` + } + if err := json.Unmarshal(respBody, &oaiResp); err != nil { + return nil, fmt.Errorf("parse response: %w", err) + } + if len(oaiResp.Choices) == 0 || oaiResp.Choices[0].Message.Content == "" { + return nil, fmt.Errorf("empty response") + } + + chatResp := &providers.ChatResponse{ + Content: oaiResp.Choices[0].Message.Content, + FinishReason: "stop", + } + if oaiResp.Usage != nil { + chatResp.Usage = &providers.Usage{ + PromptTokens: oaiResp.Usage.PromptTokens, + CompletionTokens: oaiResp.Usage.CompletionTokens, + TotalTokens: oaiResp.Usage.TotalTokens, + } + } + return chatResp, nil +} diff --git a/internal/tools/policy.go b/internal/tools/policy.go index 10ac7977..7a8bfae4 100644 --- a/internal/tools/policy.go +++ b/internal/tools/policy.go @@ -27,7 +27,9 @@ var toolGroups = map[string][]string{ "memory_search", "memory_get", "sessions_list", "sessions_history", "sessions_send", "spawn", "session_status", "cron", "message", "create_forum_topic", - "read_image", "create_image", "skill_search", "mcp_tool_search", "tts", + "read_image", "read_document", "read_audio", "read_video", + "create_image", "create_video", + "skill_search", "mcp_tool_search", "tts", "handoff", "delegate_search", "evaluate_loop", "team_tasks", "team_message", }, diff --git a/internal/tools/read_audio.go b/internal/tools/read_audio.go new file mode 100644 index 00000000..f9439118 --- /dev/null +++ b/internal/tools/read_audio.go @@ -0,0 +1,155 @@ +package tools + +import ( + "context" + "fmt" + "log/slog" + "os" + "strings" + + "github.com/nextlevelbuilder/goclaw/internal/providers" +) + +// --- Context helpers for media audio --- + +const ctxMediaAudioRefs toolContextKey = "tool_media_audio_refs" + +// WithMediaAudioRefs stores audio MediaRefs in context for read_audio tool access. +func WithMediaAudioRefs(ctx context.Context, refs []providers.MediaRef) context.Context { + return context.WithValue(ctx, ctxMediaAudioRefs, refs) +} + +// MediaAudioRefsFromCtx retrieves stored audio MediaRefs from context. +func MediaAudioRefsFromCtx(ctx context.Context) []providers.MediaRef { + v, _ := ctx.Value(ctxMediaAudioRefs).([]providers.MediaRef) + return v +} + +// --- ReadAudioTool --- + +// audioMaxBytes is the max file size for audio analysis (50MB). +const audioMaxBytes = 50 * 1024 * 1024 + +// ReadAudioTool uses an audio-capable provider to analyze audio files +// attached to the current conversation. Follows same pattern as ReadDocumentTool. +type ReadAudioTool struct { + registry *providers.Registry + mediaLoader MediaPathLoader +} + +func NewReadAudioTool(registry *providers.Registry, mediaLoader MediaPathLoader) *ReadAudioTool { + return &ReadAudioTool{registry: registry, mediaLoader: mediaLoader} +} + +func (t *ReadAudioTool) Name() string { return "read_audio" } + +func (t *ReadAudioTool) Description() string { + return "Analyze audio files (speech, music, sounds) attached to the conversation. " + + "Use when you see tags and need to transcribe, summarize, or analyze audio content. " + + "Specify what you want to extract or analyze." +} + +func (t *ReadAudioTool) Parameters() map[string]interface{} { + return map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "prompt": map[string]interface{}{ + "type": "string", + "description": "What to analyze. E.g. 'Transcribe this audio', 'Summarize the conversation', 'What language is spoken?'", + }, + "media_id": map[string]interface{}{ + "type": "string", + "description": "Optional: specific media_id from tag. If omitted, uses most recent audio.", + }, + }, + "required": []string{"prompt"}, + } +} + +func (t *ReadAudioTool) Execute(ctx context.Context, args map[string]interface{}) *Result { + prompt, _ := args["prompt"].(string) + if prompt == "" { + prompt = "Analyze this audio and describe its contents." + } + mediaID, _ := args["media_id"].(string) + + // Resolve audio file path from MediaRefs in context. + audioPath, audioMime, err := t.resolveAudioFile(ctx, mediaID) + if err != nil { + return ErrorResult(err.Error()) + } + + slog.Info("read_audio: resolved file", "path", audioPath, "mime", audioMime, "media_id", mediaID) + + // Read audio file. + data, err := os.ReadFile(audioPath) + if err != nil { + return ErrorResult(fmt.Sprintf("Failed to read audio file: %v", err)) + } + slog.Info("read_audio: file loaded", "size_bytes", len(data)) + if len(data) > audioMaxBytes { + return ErrorResult(fmt.Sprintf("Audio too large: %d bytes (max %d)", len(data), audioMaxBytes)) + } + + // Find an audio-capable provider. + provider, model, err := t.resolveAudioProviderWithConfig(ctx) + if err != nil { + return ErrorResult(err.Error()) + } + + // Try primary provider, fallback to next available on error. + resp, usedProvider, usedModel := t.callAudioProvider(ctx, provider, model, prompt, data, audioMime) + if resp == nil { + // Primary failed — try fallback providers from priority list. + slog.Warn("read_audio: primary provider failed, trying fallback", "primary", provider.Name()) + for _, fbName := range audioProviderPriority { + if fbName == provider.Name() { + continue + } + fbProvider, fbModel, fbErr := t.resolveAudioProviderByName(fbName) + if fbErr != nil { + continue + } + resp, usedProvider, usedModel = t.callAudioProvider(ctx, fbProvider, fbModel, prompt, data, audioMime) + if resp != nil { + slog.Info("read_audio: fallback succeeded", "provider", usedProvider) + break + } + } + } + if resp == nil { + return ErrorResult("Audio analysis failed: all providers returned errors") + } + + result := NewResult(resp.Content) + result.Usage = resp.Usage + result.Provider = usedProvider + result.Model = usedModel + return result +} + +// mimeFromAudioExt returns MIME type for audio file extensions. +func mimeFromAudioExt(ext string) string { + switch strings.ToLower(ext) { + case ".mp3": + return "audio/mpeg" + case ".wav": + return "audio/wav" + case ".ogg", ".oga": + return "audio/ogg" + case ".m4a": + return "audio/mp4" + case ".aac": + return "audio/aac" + case ".flac": + return "audio/flac" + case ".aiff", ".aif": + return "audio/aiff" + case ".wma": + return "audio/x-ms-wma" + case ".opus": + return "audio/opus" + default: + return "audio/mpeg" + } +} diff --git a/internal/tools/read_audio_resolve.go b/internal/tools/read_audio_resolve.go new file mode 100644 index 00000000..2da82ea0 --- /dev/null +++ b/internal/tools/read_audio_resolve.go @@ -0,0 +1,224 @@ +package tools + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "log/slog" + "path/filepath" + "strings" + "time" + + "github.com/nextlevelbuilder/goclaw/internal/providers" +) + +// audioProviderPriority is the order in which providers are tried for audio analysis. +var audioProviderPriority = []string{"gemini", "openai", "openrouter"} + +// audioModelOverrides maps provider names to preferred audio-capable models. +var audioModelOverrides = map[string]string{ + "gemini": "gemini-2.5-flash", + "openai": "gpt-4o-audio-preview", + "openrouter": "google/gemini-2.5-flash", +} + +// resolveAudioFile finds the audio file path from context MediaRefs. +func (t *ReadAudioTool) resolveAudioFile(ctx context.Context, mediaID string) (path, mime string, err error) { + if t.mediaLoader == nil { + return "", "", fmt.Errorf("no media storage configured — cannot access audio files") + } + + refs := MediaAudioRefsFromCtx(ctx) + if len(refs) == 0 { + return "", "", fmt.Errorf("no audio files available in this conversation. The user may not have sent an audio file.") + } + + var ref *providers.MediaRef + if mediaID != "" { + for i := range refs { + if refs[i].ID == mediaID { + ref = &refs[i] + break + } + } + if ref == nil { + return "", "", fmt.Errorf("audio with media_id %q not found in conversation", mediaID) + } + } else { + ref = &refs[len(refs)-1] + } + + p, err := t.mediaLoader.LoadPath(ref.ID) + if err != nil { + return "", "", fmt.Errorf("audio file not found: %v", err) + } + + mime = ref.MimeType + if mime == "" || mime == "application/octet-stream" { + mime = mimeFromAudioExt(filepath.Ext(p)) + } + + return p, mime, nil +} + +// resolveAudioProviderWithConfig checks builtin settings, then hardcoded priority. +func (t *ReadAudioTool) resolveAudioProviderWithConfig(ctx context.Context) (providers.Provider, string, error) { + if p, model, ok := t.resolveFromAudioSettings(ctx); ok { + return p, model, nil + } + return t.resolveAudioProvider() +} + +// resolveFromAudioSettings checks global builtin tool settings for provider/model config. +func (t *ReadAudioTool) resolveFromAudioSettings(ctx context.Context) (providers.Provider, string, bool) { + settings := BuiltinToolSettingsFromCtx(ctx) + if settings == nil { + return nil, "", false + } + raw, ok := settings["read_audio"] + if !ok || len(raw) == 0 { + return nil, "", false + } + var cfg struct { + Provider string `json:"provider"` + Model string `json:"model"` + } + if err := json.Unmarshal(raw, &cfg); err != nil || cfg.Provider == "" { + return nil, "", false + } + p, err := t.registry.Get(cfg.Provider) + if err != nil { + return nil, "", false + } + model := cfg.Model + if model == "" { + model = p.DefaultModel() + } + return p, model, true +} + +// resolveAudioProvider finds the first available audio-capable provider. +func (t *ReadAudioTool) resolveAudioProvider() (providers.Provider, string, error) { + for _, name := range audioProviderPriority { + p, err := t.registry.Get(name) + if err != nil { + continue + } + model := p.DefaultModel() + if override, ok := audioModelOverrides[name]; ok { + model = override + } + return p, model, nil + } + return nil, "", fmt.Errorf("no audio-capable provider available (need one of: %v)", audioProviderPriority) +} + +// resolveAudioProviderByName gets a specific provider by name and applies model override. +func (t *ReadAudioTool) resolveAudioProviderByName(name string) (providers.Provider, string, error) { + p, err := t.registry.Get(name) + if err != nil { + return nil, "", err + } + model := p.DefaultModel() + if override, ok := audioModelOverrides[name]; ok { + model = override + } + return p, model, nil +} + +// callAudioProvider sends audio to a provider for analysis. +// Gemini: uses File API (upload → poll → file_data in generateContent). +// OpenAI: uses input_audio content part in chat completions. +// Others: falls back to base64 in image_url (may not work for all). +func (t *ReadAudioTool) callAudioProvider(ctx context.Context, provider providers.Provider, model, prompt string, data []byte, mime string) (*providers.ChatResponse, string, string) { + provName := provider.Name() + + // Gemini: use File API (inlineData doesn't work for audio). + if strings.HasPrefix(provName, "gemini") { + oaiProv, ok := provider.(*providers.OpenAIProvider) + if !ok { + slog.Warn("read_audio: gemini provider is not OpenAIProvider", "provider", provName) + return nil, "", "" + } + apiKey := oaiProv.APIKey() + slog.Info("read_audio: using gemini file API", "provider", provName, "model", model, "size", len(data), "mime", mime) + resp, err := geminiFileAPICall(ctx, apiKey, model, prompt, data, mime, 120*time.Second) + if err != nil { + slog.Warn("read_audio: gemini file API call failed", "error", err) + return nil, "", "" + } + return resp, provName, model + } + + // OpenAI: use input_audio content part (supports wav, mp3). + if strings.HasPrefix(provName, "openai") { + oaiProv, ok := provider.(*providers.OpenAIProvider) + if !ok { + slog.Warn("read_audio: openai provider is not OpenAIProvider", "provider", provName) + return nil, "", "" + } + slog.Info("read_audio: using openai input_audio API", "provider", provName, "model", model, "size", len(data), "mime", mime) + resp, err := openaiAudioCall(ctx, oaiProv.APIKey(), oaiProv.APIBase(), model, prompt, data, mime) + if err != nil { + slog.Warn("read_audio: openai audio call failed", "error", err) + return nil, "", "" + } + return resp, provName, model + } + + // Other providers: try standard Chat API with base64 audio as image_url (best effort). + slog.Info("read_audio: using chat API fallback", "provider", provName, "model", model, "size", len(data)) + resp, err := provider.Chat(ctx, providers.ChatRequest{ + Messages: []providers.Message{ + { + Role: "user", + Content: prompt, + Images: []providers.ImageContent{{MimeType: mime, Data: base64.StdEncoding.EncodeToString(data)}}, + }, + }, + Model: model, + Options: map[string]interface{}{ + "max_tokens": 16384, + "temperature": 0.2, + }, + }) + if err != nil { + slog.Warn("read_audio: chat API fallback failed", "provider", provName, "error", err) + return nil, "", "" + } + return resp, provName, model +} + +// openaiAudioCall sends audio to OpenAI using the input_audio content part. +func openaiAudioCall(ctx context.Context, apiKey, baseURL, model, prompt string, data []byte, mime string) (*providers.ChatResponse, error) { + // Determine format from MIME (OpenAI supports: wav, mp3). + format := "mp3" + switch { + case strings.Contains(mime, "wav"): + format = "wav" + case strings.Contains(mime, "mp3"), strings.Contains(mime, "mpeg"): + format = "mp3" + } + + b64 := base64.StdEncoding.EncodeToString(data) + + body := map[string]interface{}{ + "model": model, + "messages": []map[string]interface{}{ + { + "role": "user", + "content": []map[string]interface{}{ + {"type": "text", "text": prompt}, + {"type": "input_audio", "input_audio": map[string]string{ + "data": b64, + "format": format, + }}, + }, + }, + }, + "max_tokens": 16384, + } + + return callOpenAICompatJSON(ctx, apiKey, baseURL, body, 120*time.Second) +} diff --git a/internal/tools/read_video.go b/internal/tools/read_video.go new file mode 100644 index 00000000..5ee9f6dd --- /dev/null +++ b/internal/tools/read_video.go @@ -0,0 +1,150 @@ +package tools + +import ( + "context" + "fmt" + "log/slog" + "os" + "strings" + + "github.com/nextlevelbuilder/goclaw/internal/providers" +) + +// --- Context helpers for media video --- + +const ctxMediaVideoRefs toolContextKey = "tool_media_video_refs" + +// WithMediaVideoRefs stores video MediaRefs in context for read_video tool access. +func WithMediaVideoRefs(ctx context.Context, refs []providers.MediaRef) context.Context { + return context.WithValue(ctx, ctxMediaVideoRefs, refs) +} + +// MediaVideoRefsFromCtx retrieves stored video MediaRefs from context. +func MediaVideoRefsFromCtx(ctx context.Context) []providers.MediaRef { + v, _ := ctx.Value(ctxMediaVideoRefs).([]providers.MediaRef) + return v +} + +// --- ReadVideoTool --- + +// videoMaxBytes is the max file size for video analysis (100MB). +const videoMaxBytes = 100 * 1024 * 1024 + +// ReadVideoTool uses a video-capable provider to analyze video files +// attached to the current conversation. Follows same pattern as ReadAudioTool. +type ReadVideoTool struct { + registry *providers.Registry + mediaLoader MediaPathLoader +} + +func NewReadVideoTool(registry *providers.Registry, mediaLoader MediaPathLoader) *ReadVideoTool { + return &ReadVideoTool{registry: registry, mediaLoader: mediaLoader} +} + +func (t *ReadVideoTool) Name() string { return "read_video" } + +func (t *ReadVideoTool) Description() string { + return "Analyze video files attached to the conversation. " + + "Use when you see tags and need to describe, summarize, or analyze video content. " + + "Specify what you want to extract or analyze." +} + +func (t *ReadVideoTool) Parameters() map[string]interface{} { + return map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "prompt": map[string]interface{}{ + "type": "string", + "description": "What to analyze. E.g. 'Describe what happens in this video', 'Summarize the key scenes', 'What text appears on screen?'", + }, + "media_id": map[string]interface{}{ + "type": "string", + "description": "Optional: specific media_id from tag. If omitted, uses most recent video.", + }, + }, + "required": []string{"prompt"}, + } +} + +func (t *ReadVideoTool) Execute(ctx context.Context, args map[string]interface{}) *Result { + prompt, _ := args["prompt"].(string) + if prompt == "" { + prompt = "Analyze this video and describe its contents." + } + mediaID, _ := args["media_id"].(string) + + videoPath, videoMime, err := t.resolveVideoFile(ctx, mediaID) + if err != nil { + return ErrorResult(err.Error()) + } + + slog.Info("read_video: resolved file", "path", videoPath, "mime", videoMime, "media_id", mediaID) + + data, err := os.ReadFile(videoPath) + if err != nil { + return ErrorResult(fmt.Sprintf("Failed to read video file: %v", err)) + } + slog.Info("read_video: file loaded", "size_bytes", len(data)) + if len(data) > videoMaxBytes { + return ErrorResult(fmt.Sprintf("Video too large: %d bytes (max %d)", len(data), videoMaxBytes)) + } + + provider, model, err := t.resolveVideoProviderWithConfig(ctx) + if err != nil { + return ErrorResult(err.Error()) + } + + resp, usedProvider, usedModel := t.callVideoProvider(ctx, provider, model, prompt, data, videoMime) + if resp == nil { + slog.Warn("read_video: primary provider failed, trying fallback", "primary", provider.Name()) + for _, fbName := range videoProviderPriority { + if fbName == provider.Name() { + continue + } + fbProvider, fbModel, fbErr := t.resolveVideoProviderByName(fbName) + if fbErr != nil { + continue + } + resp, usedProvider, usedModel = t.callVideoProvider(ctx, fbProvider, fbModel, prompt, data, videoMime) + if resp != nil { + slog.Info("read_video: fallback succeeded", "provider", usedProvider) + break + } + } + } + if resp == nil { + return ErrorResult("Video analysis failed: all providers returned errors") + } + + result := NewResult(resp.Content) + result.Usage = resp.Usage + result.Provider = usedProvider + result.Model = usedModel + return result +} + +// mimeFromVideoExt returns MIME type for video file extensions. +func mimeFromVideoExt(ext string) string { + switch strings.ToLower(ext) { + case ".mp4": + return "video/mp4" + case ".webm": + return "video/webm" + case ".mov": + return "video/quicktime" + case ".avi": + return "video/x-msvideo" + case ".mkv": + return "video/x-matroska" + case ".wmv": + return "video/x-ms-wmv" + case ".flv": + return "video/x-flv" + case ".3gp": + return "video/3gpp" + case ".mpeg", ".mpg": + return "video/mpeg" + default: + return "video/mp4" + } +} diff --git a/internal/tools/read_video_resolve.go b/internal/tools/read_video_resolve.go new file mode 100644 index 00000000..ef905d73 --- /dev/null +++ b/internal/tools/read_video_resolve.go @@ -0,0 +1,174 @@ +package tools + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "log/slog" + "path/filepath" + "strings" + "time" + + "github.com/nextlevelbuilder/goclaw/internal/providers" +) + +// videoProviderPriority is the order in which providers are tried for video analysis. +// OpenAI excluded — no native video upload in chat completions. +var videoProviderPriority = []string{"gemini", "openrouter"} + +// videoModelOverrides maps provider names to preferred video-capable models. +var videoModelOverrides = map[string]string{ + "gemini": "gemini-2.5-flash", + "openrouter": "google/gemini-2.5-flash", +} + +// resolveVideoFile finds the video file path from context MediaRefs. +func (t *ReadVideoTool) resolveVideoFile(ctx context.Context, mediaID string) (path, mime string, err error) { + if t.mediaLoader == nil { + return "", "", fmt.Errorf("no media storage configured — cannot access video files") + } + + refs := MediaVideoRefsFromCtx(ctx) + if len(refs) == 0 { + return "", "", fmt.Errorf("no video files available in this conversation. The user may not have sent a video file.") + } + + var ref *providers.MediaRef + if mediaID != "" { + for i := range refs { + if refs[i].ID == mediaID { + ref = &refs[i] + break + } + } + if ref == nil { + return "", "", fmt.Errorf("video with media_id %q not found in conversation", mediaID) + } + } else { + ref = &refs[len(refs)-1] + } + + p, err := t.mediaLoader.LoadPath(ref.ID) + if err != nil { + return "", "", fmt.Errorf("video file not found: %v", err) + } + + mime = ref.MimeType + if mime == "" || mime == "application/octet-stream" { + mime = mimeFromVideoExt(filepath.Ext(p)) + } + + return p, mime, nil +} + +// resolveVideoProviderWithConfig checks builtin settings, then hardcoded priority. +func (t *ReadVideoTool) resolveVideoProviderWithConfig(ctx context.Context) (providers.Provider, string, error) { + if p, model, ok := t.resolveFromVideoSettings(ctx); ok { + return p, model, nil + } + return t.resolveVideoProvider() +} + +// resolveFromVideoSettings checks global builtin tool settings for provider/model config. +func (t *ReadVideoTool) resolveFromVideoSettings(ctx context.Context) (providers.Provider, string, bool) { + settings := BuiltinToolSettingsFromCtx(ctx) + if settings == nil { + return nil, "", false + } + raw, ok := settings["read_video"] + if !ok || len(raw) == 0 { + return nil, "", false + } + var cfg struct { + Provider string `json:"provider"` + Model string `json:"model"` + } + if err := json.Unmarshal(raw, &cfg); err != nil || cfg.Provider == "" { + return nil, "", false + } + p, err := t.registry.Get(cfg.Provider) + if err != nil { + return nil, "", false + } + model := cfg.Model + if model == "" { + model = p.DefaultModel() + } + return p, model, true +} + +// resolveVideoProvider finds the first available video-capable provider. +func (t *ReadVideoTool) resolveVideoProvider() (providers.Provider, string, error) { + for _, name := range videoProviderPriority { + p, err := t.registry.Get(name) + if err != nil { + continue + } + model := p.DefaultModel() + if override, ok := videoModelOverrides[name]; ok { + model = override + } + return p, model, nil + } + return nil, "", fmt.Errorf("no video-capable provider available (need one of: %v)", videoProviderPriority) +} + +// resolveVideoProviderByName gets a specific provider by name and applies model override. +func (t *ReadVideoTool) resolveVideoProviderByName(name string) (providers.Provider, string, error) { + p, err := t.registry.Get(name) + if err != nil { + return nil, "", err + } + model := p.DefaultModel() + if override, ok := videoModelOverrides[name]; ok { + model = override + } + return p, model, nil +} + +// callVideoProvider sends video to a provider for analysis. +// Gemini: uses File API (upload → poll → file_data in generateContent). +// Others: falls back to base64 in image_url (OpenRouter routes to Gemini which handles video). +func (t *ReadVideoTool) callVideoProvider(ctx context.Context, provider providers.Provider, model, prompt string, data []byte, mime string) (*providers.ChatResponse, string, string) { + provName := provider.Name() + + // Gemini: use File API. + if strings.HasPrefix(provName, "gemini") { + oaiProv, ok := provider.(*providers.OpenAIProvider) + if !ok { + slog.Warn("read_video: gemini provider is not OpenAIProvider", "provider", provName) + return nil, "", "" + } + apiKey := oaiProv.APIKey() + slog.Info("read_video: using gemini file API", "provider", provName, "model", model, "size", len(data), "mime", mime) + resp, err := geminiFileAPICall(ctx, apiKey, model, prompt, data, mime, 180*time.Second) + if err != nil { + slog.Warn("read_video: gemini file API call failed", "error", err) + return nil, "", "" + } + return resp, provName, model + } + + // Other providers: try standard Chat API with base64 as image_url (best effort). + slog.Info("read_video: using chat API fallback", "provider", provName, "model", model, "size", len(data)) + resp, err := provider.Chat(ctx, providers.ChatRequest{ + Messages: []providers.Message{ + { + Role: "user", + Content: prompt, + Images: []providers.ImageContent{{MimeType: mime, Data: base64.StdEncoding.EncodeToString(data)}}, + }, + }, + Model: model, + Options: map[string]interface{}{ + "max_tokens": 16384, + "temperature": 0.2, + }, + }) + if err != nil { + slog.Warn("read_video: chat API fallback failed", "provider", provName, "error", err) + return nil, "", "" + } + return resp, provName, model +} diff --git a/ui/web/src/pages/builtin-tools/builtin-tool-settings-dialog.tsx b/ui/web/src/pages/builtin-tools/builtin-tool-settings-dialog.tsx index b07b1830..2de54381 100644 --- a/ui/web/src/pages/builtin-tools/builtin-tool-settings-dialog.tsx +++ b/ui/web/src/pages/builtin-tools/builtin-tool-settings-dialog.tsx @@ -30,7 +30,10 @@ interface Props { onSave: (name: string, settings: Record) => Promise; } -const MEDIA_TOOLS = new Set(["read_image", "create_image", "read_document"]); +const MEDIA_TOOLS = new Set([ + "read_image", "read_document", "read_audio", "read_video", + "create_image", "create_video", +]); export function BuiltinToolSettingsDialog({ tool, open, onOpenChange, onSave }: Props) { const isMedia = tool ? MEDIA_TOOLS.has(tool.name) : false;