Files
goclaw/internal/tools/create_image_byteplus.go
T
thotam a5a853f461 feat(tools): image reference processing and native provider support (#1251)
* feat(tools): implement image reference processing and native provider support

- Support OpenAI image edits via both Multipart form-data and JSON payloads
- Automatically append reference image descriptions to prompt under [Reference Image Roles]
- Support downloading image URLs for Gemini native image generation
- Deduplicate reference images to optimize API request size
- Add unit tests for Codex, DashScope, MiniMax, BytePlus and local/remote path resolution

* fix(tools): SSRF-guard reference-image URL downloads in create_image

downloadImageBytes fetched caller-supplied ref_images[].url with a plain
http.Client and unbounded io.ReadAll — no SSRF validation, redirect
policy, or size cap, letting the gateway dial loopback/private/metadata
hosts or read arbitrarily large responses.

- Validate the URL via security.Validate and pin the resolved IP, then
  download through security.NewSafeClient (pinned dial, no redirects).
- Cap the response with a bounded read (refImageMaxBytes, 20 MB).
- Reject non-HTTP(S) reference URLs up front (file://, data:, gopher://)
  so provider-forwarded URLs stay HTTP(S)-only; document the trust
  boundary between gateway-side fetch and provider-forwarded URLs.
- Add regression tests: blocked loopback/private/metadata, unfollowed
  redirect, oversized response, and non-http(s) scheme rejection.
2026-06-21 18:27:51 +07:00

135 lines
3.9 KiB
Go

package tools
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"strings"
"github.com/nextlevelbuilder/goclaw/internal/providers"
)
// bytePlusImageEndpoint derives the Seedream image generation endpoint from the stored api_base.
// Media endpoints always use the standard /api/v3 path (not /api/coding/v3).
func bytePlusImageEndpoint(apiBase string) string {
return bytePlusMediaBase(apiBase) + "/images/generations"
}
// bytePlusMediaBase extracts the host and returns the standard /api/v3 media base.
// Strips /chat/completions, /coding/v3, etc. to always use /api/v3 for media endpoints.
func bytePlusMediaBase(apiBase string) string {
base := strings.TrimRight(apiBase, "/")
base = strings.TrimSuffix(base, "/chat/completions")
// Strip any versioned path to rebuild consistently
for _, suffix := range []string{"/api/coding/v3", "/api/v3", "/v3"} {
if before, ok := strings.CutSuffix(base, suffix); ok {
return before + "/api/v3"
}
}
return base + "/api/v3"
}
// callBytePlusImageGen calls the BytePlus Seedream image generation API.
// Seedream returns results synchronously (no async polling needed).
// Endpoint: POST /api/v3/images/generations
func callBytePlusImageGen(ctx context.Context, apiKey, apiBase, model, prompt string, params map[string]any) ([]byte, *providers.Usage, error) {
size := aspectRatioToBytePlusSize(params)
endpoint := bytePlusImageEndpoint(apiBase)
body := map[string]any{
"model": model,
"prompt": prompt,
"size": size,
"response_format": "url",
}
if rawImgs, ok := params["ref_images"]; ok {
if refImgs, ok := rawImgs.([]*referenceImage); ok && len(refImgs) > 0 {
if len(refImgs) > 1 {
slog.Warn("byteplus image gen: provider only supports 1 reference image, using the first one", "count", len(refImgs))
}
refImg := refImgs[0]
var refURL string
if refImg.URL != "" {
refURL = refImg.URL
} else {
refMime := refImg.MimeType
if refMime == "" {
refMime = "image/png"
}
refURL = fmt.Sprintf("data:%s;base64,%s", refMime, refImg.Base64)
}
body["image"] = refURL
}
}
jsonBody, err := json.Marshal(body)
if err != nil {
return nil, nil, fmt.Errorf("marshal request: %w", err)
}
slog.Info("create_image: calling BytePlus Seedream API", "model", model, "size", size)
req, err := http.NewRequestWithContext(ctx, "POST", endpoint, bytes.NewReader(jsonBody))
if err != nil {
return nil, nil, fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+apiKey)
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, nil, fmt.Errorf("http request: %w", err)
}
defer resp.Body.Close()
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return nil, nil, fmt.Errorf("read response: %w", err)
}
if resp.StatusCode != http.StatusOK {
return nil, nil, fmt.Errorf("API error %d: %s", resp.StatusCode, truncateBytes(respBody, 500))
}
var result struct {
Data []struct {
URL string `json:"url"`
} `json:"data"`
}
if err := json.Unmarshal(respBody, &result); err != nil {
return nil, nil, fmt.Errorf("parse response: %w", err)
}
if len(result.Data) == 0 || result.Data[0].URL == "" {
return nil, nil, fmt.Errorf("no image URL in BytePlus response: %s", truncateBytes(respBody, 300))
}
return downloadImageURL(ctx, result.Data[0].URL)
}
// aspectRatioToBytePlusSize converts aspect_ratio to BytePlus size format.
// Seedream supports "1k", "2K", "4K" or "WIDTHxHEIGHT".
func aspectRatioToBytePlusSize(params map[string]any) string {
if s := GetParamString(params, "size", ""); s != "" {
return s
}
ar := GetParamString(params, "aspect_ratio", "1:1")
switch ar {
case "16:9":
return "1280x720"
case "9:16":
return "720x1280"
case "4:3":
return "1024x768"
case "3:4":
return "768x1024"
default:
return "1024x1024"
}
}