mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-07-30 20:23:02 +00:00
Root cause: create_image tool only set ForLLM:"MEDIA:path" but never
populated result.Media. The main agent loop parses MEDIA: prefix via
parseMediaResult(), but the subagent exec loop only checked result.Media
— so media paths were silently lost for all subagent/spawn workflows.
This caused the entire downstream pipeline (task.Media → AnnounceQueue →
PublishInbound → ContentSuffix → session) to receive empty media, making
images invisible in WS chat despite Telegram working fine.
Fixes:
- create_image.go: set result.Media = []string{imagePath}
- subagent_exec.go: add MEDIA: prefix fallback parsing as safety net
494 lines
15 KiB
Go
494 lines
15 KiB
Go
package tools
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
|
)
|
|
|
|
// credentialProvider is a narrow interface for providers that expose API credentials.
|
|
type credentialProvider interface {
|
|
APIKey() string
|
|
APIBase() string
|
|
}
|
|
|
|
// imageGenProviderPriority is the default order for image generation providers.
|
|
var imageGenProviderPriority = []string{"openrouter", "gemini", "openai"}
|
|
|
|
// imageGenModelDefaults maps provider names to default image generation models.
|
|
var imageGenModelDefaults = map[string]string{
|
|
"openrouter": "google/gemini-2.5-flash-image",
|
|
"openai": "dall-e-3",
|
|
"gemini": "gemini-2.5-flash-image",
|
|
}
|
|
|
|
// CreateImageTool generates images using an image generation API.
|
|
// Uses OpenRouter (Gemini image model) or OpenAI (DALL-E) via per-agent ImageGenConfig.
|
|
type CreateImageTool struct {
|
|
registry *providers.Registry
|
|
}
|
|
|
|
func NewCreateImageTool(registry *providers.Registry) *CreateImageTool {
|
|
return &CreateImageTool{registry: registry}
|
|
}
|
|
|
|
func (t *CreateImageTool) Name() string { return "create_image" }
|
|
|
|
func (t *CreateImageTool) Description() string {
|
|
return "Generate an image from a text description using an image generation model. Returns a MEDIA: path to the generated image file."
|
|
}
|
|
|
|
func (t *CreateImageTool) Parameters() map[string]interface{} {
|
|
return map[string]interface{}{
|
|
"type": "object",
|
|
"properties": map[string]interface{}{
|
|
"prompt": map[string]interface{}{
|
|
"type": "string",
|
|
"description": "Text description of the image to generate.",
|
|
},
|
|
"aspect_ratio": map[string]interface{}{
|
|
"type": "string",
|
|
"description": "Aspect ratio: '1:1' (default), '3:4', '4:3', '9:16', '16:9'.",
|
|
},
|
|
},
|
|
"required": []string{"prompt"},
|
|
}
|
|
}
|
|
|
|
func (t *CreateImageTool) Execute(ctx context.Context, args map[string]interface{}) *Result {
|
|
prompt, _ := args["prompt"].(string)
|
|
if prompt == "" {
|
|
return ErrorResult("prompt is required")
|
|
}
|
|
aspectRatio, _ := args["aspect_ratio"].(string)
|
|
if aspectRatio == "" {
|
|
aspectRatio = "1:1"
|
|
}
|
|
|
|
// Resolve provider from per-agent config or defaults
|
|
providerName, model := t.resolveConfig(ctx)
|
|
|
|
p, err := t.registry.Get(providerName)
|
|
if err != nil {
|
|
return ErrorResult(fmt.Sprintf("image generation provider %q not available", providerName))
|
|
}
|
|
|
|
cp, ok := p.(credentialProvider)
|
|
if !ok {
|
|
return ErrorResult(fmt.Sprintf("provider %q does not expose API credentials for image generation", providerName))
|
|
}
|
|
|
|
slog.Info("create_image: calling image generation API",
|
|
"provider", providerName, "model", model, "aspect_ratio", aspectRatio)
|
|
|
|
// Route to the correct image generation endpoint per provider:
|
|
// - gemini: native Gemini generateContent API (responseModalities)
|
|
// - openrouter: OpenAI-compat /chat/completions with modalities
|
|
// - others (openai, etc.): /images/generations
|
|
var imageBytes []byte
|
|
var usage *providers.Usage
|
|
switch providerName {
|
|
case "gemini":
|
|
var genErr error
|
|
imageBytes, usage, genErr = t.callGeminiNativeImageGen(ctx, cp.APIKey(), cp.APIBase(), model, prompt)
|
|
err = genErr
|
|
case "openrouter":
|
|
var genErr error
|
|
imageBytes, usage, genErr = t.callImageGenAPI(ctx, cp.APIKey(), cp.APIBase(), model, prompt, aspectRatio)
|
|
err = genErr
|
|
default:
|
|
var genErr error
|
|
imageBytes, usage, genErr = t.callStandardImageGenAPI(ctx, cp.APIKey(), cp.APIBase(), model, prompt)
|
|
err = genErr
|
|
}
|
|
if err != nil {
|
|
return ErrorResult(fmt.Sprintf("image generation failed: %v", err))
|
|
}
|
|
|
|
// Save to workspace under date-based folder (e.g. generated/2026-03-02/)
|
|
workspace := ToolWorkspaceFromCtx(ctx)
|
|
if workspace == "" {
|
|
workspace = os.TempDir()
|
|
}
|
|
dateDir := filepath.Join(workspace, "generated", time.Now().Format("2006-01-02"))
|
|
if err := os.MkdirAll(dateDir, 0755); err != nil {
|
|
return ErrorResult(fmt.Sprintf("failed to create output directory: %v", err))
|
|
}
|
|
imagePath := filepath.Join(dateDir, fmt.Sprintf("goclaw_gen_%d.png", time.Now().UnixNano()))
|
|
if err := os.WriteFile(imagePath, imageBytes, 0644); err != nil {
|
|
return ErrorResult(fmt.Sprintf("failed to save generated image: %v", err))
|
|
}
|
|
|
|
result := &Result{ForLLM: fmt.Sprintf("MEDIA:%s", imagePath)}
|
|
result.Media = []string{imagePath}
|
|
result.Deliverable = fmt.Sprintf("[Generated image: %s]\nPrompt: %s", filepath.Base(imagePath), prompt)
|
|
result.Provider = providerName
|
|
result.Model = model
|
|
if usage != nil {
|
|
result.Usage = usage
|
|
}
|
|
return result
|
|
}
|
|
|
|
// resolveConfig returns the provider name and model to use for image generation.
|
|
func (t *CreateImageTool) resolveConfig(ctx context.Context) (providerName, model string) {
|
|
// 1. Check per-agent ImageGenConfig from context (highest priority)
|
|
if cfg := ImageGenConfigFromCtx(ctx); cfg != nil {
|
|
if cfg.Provider != "" {
|
|
providerName = cfg.Provider
|
|
}
|
|
if cfg.Model != "" {
|
|
model = cfg.Model
|
|
}
|
|
}
|
|
|
|
// 2. Check global builtin_tools.settings (DB defaults)
|
|
if providerName == "" || model == "" {
|
|
if settings := BuiltinToolSettingsFromCtx(ctx); settings != nil {
|
|
if raw, ok := settings["create_image"]; ok && len(raw) > 0 {
|
|
var cfg struct {
|
|
Provider string `json:"provider"`
|
|
Model string `json:"model"`
|
|
}
|
|
if json.Unmarshal(raw, &cfg) == nil && cfg.Provider != "" {
|
|
// DB settings are a provider+model pair — only use if provider is available
|
|
if _, err := t.registry.Get(cfg.Provider); err == nil {
|
|
if providerName == "" {
|
|
providerName = cfg.Provider
|
|
}
|
|
if model == "" && cfg.Model != "" {
|
|
model = cfg.Model
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// 3. If provider not set, find first available from priority list
|
|
if providerName == "" {
|
|
for _, name := range imageGenProviderPriority {
|
|
if _, err := t.registry.Get(name); err == nil {
|
|
providerName = name
|
|
break
|
|
}
|
|
}
|
|
}
|
|
if providerName == "" {
|
|
providerName = "openrouter" // fallback even if unavailable (error handled later)
|
|
}
|
|
|
|
// 4. If model not set, use default for this provider
|
|
if model == "" {
|
|
if m, ok := imageGenModelDefaults[providerName]; ok {
|
|
model = m
|
|
}
|
|
}
|
|
|
|
return providerName, model
|
|
}
|
|
|
|
// callImageGenAPI calls the OpenAI-compatible image generation endpoint.
|
|
// Works with OpenRouter (modalities: ["image","text"]) and OpenAI (/images/generations).
|
|
func (t *CreateImageTool) callImageGenAPI(ctx context.Context, apiKey, apiBase, model, prompt, aspectRatio string) ([]byte, *providers.Usage, error) {
|
|
// OpenRouter / OpenAI-compat: use chat completions with modalities
|
|
body := map[string]interface{}{
|
|
"model": model,
|
|
"messages": []map[string]interface{}{
|
|
{"role": "user", "content": prompt},
|
|
},
|
|
"modalities": []string{"image", "text"},
|
|
}
|
|
if aspectRatio != "" && aspectRatio != "1:1" {
|
|
body["image_config"] = map[string]interface{}{
|
|
"aspect_ratio": aspectRatio,
|
|
}
|
|
}
|
|
|
|
jsonBody, err := json.Marshal(body)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("marshal request: %w", err)
|
|
}
|
|
|
|
url := strings.TrimRight(apiBase, "/") + "/chat/completions"
|
|
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBody))
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("create request: %w", err)
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set("Authorization", "Bearer "+apiKey)
|
|
|
|
client := &http.Client{Timeout: 120 * time.Second}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("http request: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
respBody, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("read response: %w", err)
|
|
}
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return nil, nil, fmt.Errorf("API error %d: %s", resp.StatusCode, truncateBytes(respBody, 500))
|
|
}
|
|
|
|
return t.parseImageResponse(respBody)
|
|
}
|
|
|
|
// callStandardImageGenAPI uses the /images/generations endpoint (Gemini, OpenAI, and compatible providers).
|
|
// This is the standard OpenAI-compatible image generation endpoint that returns b64_json data.
|
|
func (t *CreateImageTool) callStandardImageGenAPI(ctx context.Context, apiKey, apiBase, model, prompt string) ([]byte, *providers.Usage, error) {
|
|
body := map[string]interface{}{
|
|
"model": model,
|
|
"prompt": prompt,
|
|
"n": 1,
|
|
"response_format": "b64_json",
|
|
}
|
|
|
|
jsonBody, err := json.Marshal(body)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("marshal request: %w", err)
|
|
}
|
|
|
|
url := strings.TrimRight(apiBase, "/") + "/images/generations"
|
|
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBody))
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("create request: %w", err)
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set("Authorization", "Bearer "+apiKey)
|
|
|
|
client := &http.Client{Timeout: 120 * time.Second}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("http request: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
respBody, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("read response: %w", err)
|
|
}
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return nil, nil, fmt.Errorf("API error %d: %s", resp.StatusCode, truncateBytes(respBody, 500))
|
|
}
|
|
|
|
// Parse OpenAI-compat images/generations response: {data: [{b64_json: "..."}]}
|
|
var imgResp struct {
|
|
Data []struct {
|
|
B64JSON string `json:"b64_json"`
|
|
} `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(respBody, &imgResp); err != nil {
|
|
return nil, nil, fmt.Errorf("parse response: %w", err)
|
|
}
|
|
if len(imgResp.Data) == 0 || imgResp.Data[0].B64JSON == "" {
|
|
return nil, nil, fmt.Errorf("no image data in response")
|
|
}
|
|
|
|
imageBytes, err := base64.StdEncoding.DecodeString(imgResp.Data[0].B64JSON)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("decode base64: %w", err)
|
|
}
|
|
|
|
return imageBytes, nil, nil
|
|
}
|
|
|
|
// callGeminiNativeImageGen uses the native Gemini generateContent API with responseModalities.
|
|
// Gemini image models (gemini-2.5-flash-image, gemini-3.1-flash-image-preview) require this
|
|
// endpoint — they don't support the OpenAI-compat /images/generations or /chat/completions.
|
|
func (t *CreateImageTool) callGeminiNativeImageGen(ctx context.Context, apiKey, apiBase, model, prompt string) ([]byte, *providers.Usage, error) {
|
|
// Derive native Gemini base from OpenAI-compat base (strip /openai suffix)
|
|
nativeBase := strings.TrimRight(apiBase, "/")
|
|
nativeBase = strings.TrimSuffix(nativeBase, "/openai")
|
|
|
|
url := fmt.Sprintf("%s/models/%s:generateContent?key=%s", nativeBase, model, apiKey)
|
|
|
|
body := map[string]interface{}{
|
|
"contents": []map[string]interface{}{
|
|
{"parts": []map[string]interface{}{{"text": prompt}}},
|
|
},
|
|
"generationConfig": map[string]interface{}{
|
|
"responseModalities": []string{"TEXT", "IMAGE"},
|
|
},
|
|
}
|
|
|
|
jsonBody, err := json.Marshal(body)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("marshal request: %w", err)
|
|
}
|
|
|
|
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(jsonBody))
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("create request: %w", err)
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
|
|
client := &http.Client{Timeout: 120 * time.Second}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("http request: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
respBody, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("read response: %w", err)
|
|
}
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return nil, nil, fmt.Errorf("API error %d: %s", resp.StatusCode, truncateBytes(respBody, 500))
|
|
}
|
|
|
|
// Parse native Gemini response: {candidates: [{content: {parts: [{inlineData: {mimeType, data}}]}}]}
|
|
var gemResp struct {
|
|
Candidates []struct {
|
|
Content struct {
|
|
Parts []struct {
|
|
InlineData *struct {
|
|
MimeType string `json:"mimeType"`
|
|
Data string `json:"data"`
|
|
} `json:"inlineData"`
|
|
Text string `json:"text"`
|
|
} `json:"parts"`
|
|
} `json:"content"`
|
|
} `json:"candidates"`
|
|
UsageMetadata *struct {
|
|
PromptTokenCount int `json:"promptTokenCount"`
|
|
CandidatesTokenCount int `json:"candidatesTokenCount"`
|
|
TotalTokenCount int `json:"totalTokenCount"`
|
|
} `json:"usageMetadata"`
|
|
}
|
|
if err := json.Unmarshal(respBody, &gemResp); err != nil {
|
|
return nil, nil, fmt.Errorf("parse response: %w", err)
|
|
}
|
|
|
|
// Extract first image from parts
|
|
for _, cand := range gemResp.Candidates {
|
|
for _, part := range cand.Content.Parts {
|
|
if part.InlineData != nil && strings.HasPrefix(part.InlineData.MimeType, "image/") {
|
|
imageBytes, err := base64.StdEncoding.DecodeString(part.InlineData.Data)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("decode base64: %w", err)
|
|
}
|
|
var usage *providers.Usage
|
|
if gemResp.UsageMetadata != nil {
|
|
usage = &providers.Usage{
|
|
PromptTokens: gemResp.UsageMetadata.PromptTokenCount,
|
|
CompletionTokens: gemResp.UsageMetadata.CandidatesTokenCount,
|
|
TotalTokens: gemResp.UsageMetadata.TotalTokenCount,
|
|
}
|
|
}
|
|
return imageBytes, usage, nil
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil, nil, fmt.Errorf("no image data in Gemini response")
|
|
}
|
|
|
|
// parseImageResponse extracts base64 image data from the OpenAI-compat chat response.
|
|
// Looks for images in choices[0].message.content (multipart) or choices[0].message.images.
|
|
func (t *CreateImageTool) parseImageResponse(respBody []byte) ([]byte, *providers.Usage, error) {
|
|
var resp struct {
|
|
Choices []struct {
|
|
Message struct {
|
|
Content interface{} `json:"content"`
|
|
Images []struct {
|
|
ImageURL struct {
|
|
URL string `json:"url"`
|
|
} `json:"image_url"`
|
|
} `json:"images"`
|
|
} `json:"message"`
|
|
} `json:"choices"`
|
|
Usage *struct {
|
|
PromptTokens int `json:"prompt_tokens"`
|
|
CompletionTokens int `json:"completion_tokens"`
|
|
TotalTokens int `json:"total_tokens"`
|
|
} `json:"usage"`
|
|
}
|
|
|
|
if err := json.Unmarshal(respBody, &resp); err != nil {
|
|
return nil, nil, fmt.Errorf("parse response: %w", err)
|
|
}
|
|
|
|
if len(resp.Choices) == 0 {
|
|
return nil, nil, fmt.Errorf("no choices in response")
|
|
}
|
|
|
|
msg := resp.Choices[0].Message
|
|
|
|
// Try images array first (OpenRouter format)
|
|
for _, img := range msg.Images {
|
|
if imageBytes, err := decodeDataURL(img.ImageURL.URL); err == nil {
|
|
return imageBytes, convertUsage(resp.Usage), nil
|
|
}
|
|
}
|
|
|
|
// Try multipart content array (some providers return content as array of parts)
|
|
if parts, ok := msg.Content.([]interface{}); ok {
|
|
for _, part := range parts {
|
|
if m, ok := part.(map[string]interface{}); ok {
|
|
if m["type"] == "image_url" {
|
|
if imgURL, ok := m["image_url"].(map[string]interface{}); ok {
|
|
if url, ok := imgURL["url"].(string); ok {
|
|
if imageBytes, err := decodeDataURL(url); err == nil {
|
|
return imageBytes, convertUsage(resp.Usage), nil
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil, nil, fmt.Errorf("no image data found in response")
|
|
}
|
|
|
|
// decodeDataURL decodes a data:image/...;base64,... URL into raw bytes.
|
|
func decodeDataURL(dataURL string) ([]byte, error) {
|
|
// Format: data:image/png;base64,iVBORw0KGgo...
|
|
idx := strings.Index(dataURL, ";base64,")
|
|
if idx < 0 {
|
|
return nil, fmt.Errorf("not a base64 data URL")
|
|
}
|
|
b64 := dataURL[idx+8:]
|
|
return base64.StdEncoding.DecodeString(b64)
|
|
}
|
|
|
|
func convertUsage(u *struct {
|
|
PromptTokens int `json:"prompt_tokens"`
|
|
CompletionTokens int `json:"completion_tokens"`
|
|
TotalTokens int `json:"total_tokens"`
|
|
}) *providers.Usage {
|
|
if u == nil {
|
|
return nil
|
|
}
|
|
return &providers.Usage{
|
|
PromptTokens: u.PromptTokens,
|
|
CompletionTokens: u.CompletionTokens,
|
|
TotalTokens: u.TotalTokens,
|
|
}
|
|
}
|
|
|
|
func truncateBytes(b []byte, max int) string {
|
|
if len(b) <= max {
|
|
return string(b)
|
|
}
|
|
return string(b[:max]) + "..."
|
|
}
|