Files
goclaw/internal/providers/native_image.go
T
thotam a5a853f461 feat(tools): image reference processing and native provider support (#1251)
* feat(tools): implement image reference processing and native provider support

- Support OpenAI image edits via both Multipart form-data and JSON payloads
- Automatically append reference image descriptions to prompt under [Reference Image Roles]
- Support downloading image URLs for Gemini native image generation
- Deduplicate reference images to optimize API request size
- Add unit tests for Codex, DashScope, MiniMax, BytePlus and local/remote path resolution

* fix(tools): SSRF-guard reference-image URL downloads in create_image

downloadImageBytes fetched caller-supplied ref_images[].url with a plain
http.Client and unbounded io.ReadAll — no SSRF validation, redirect
policy, or size cap, letting the gateway dial loopback/private/metadata
hosts or read arbitrarily large responses.

- Validate the URL via security.Validate and pin the resolved IP, then
  download through security.NewSafeClient (pinned dial, no redirects).
- Cap the response with a bounded read (refImageMaxBytes, 20 MB).
- Reject non-HTTP(S) reference URLs up front (file://, data:, gopher://)
  so provider-forwarded URLs stay HTTP(S)-only; document the trust
  boundary between gateway-side fetch and provider-forwarded URLs.
- Add regression tests: blocked loopback/private/metadata, unfollowed
  redirect, oversized response, and non-http(s) scheme rejection.
2026-06-21 18:27:51 +07:00

107 lines
3.5 KiB
Go

package providers
import (
"context"
"fmt"
)
// NativeImageProvider is implemented by OAuth-backed providers whose upstream
// exposes an image_generation native tool (ChatGPT Responses API style).
// create_image routes through this interface when the chain resolves to such
// a provider, bypassing the credentialProvider (APIKey/APIBase) path.
type NativeImageProvider interface {
GenerateImage(ctx context.Context, req NativeImageRequest) (*NativeImageResult, error)
}
// DefaultImageModel is the image model used by the Responses API image_generation
// tool when the caller does not specify one. gpt-image-2 is the current (2026-Q2)
// quality baseline; gpt-image-1.5 is available as a legacy fallback.
const DefaultImageModel = "gpt-image-2"
// allowedImageModels enumerates the image models the native ChatGPT Responses API
// image_generation tool will accept. Constraining to this whitelist prevents
// silent upstream rejections from arbitrary model names (e.g. "dall-e-3") and
// keeps the PR's motivation — gpt-image-2 quality — as the default everywhere.
var allowedImageModels = map[string]bool{
"gpt-image-2": true, // default — latest quality
"gpt-image-1.5": true, // legacy fallback
}
// ValidateImageModel returns the model to use, or an error if the caller
// supplied an unsupported value. Empty input returns DefaultImageModel.
func ValidateImageModel(model string) (string, error) {
if model == "" {
return DefaultImageModel, nil
}
if !allowedImageModels[model] {
return "", fmt.Errorf("unsupported image model %q; allowed: gpt-image-2 (default), gpt-image-1.5 (legacy)", model)
}
return model, nil
}
// NativeImageRequest describes a single image generation request.
type NativeImageRequest struct {
// Model is the parent LLM model for the Responses API call (e.g. "gpt-5.5").
// NOT the image model — see ImageModel below.
// If empty, the provider uses its own default LLM model.
Model string
// ImageModel is the image-generation model attached to the image_generation
// tool (e.g. "gpt-image-2"). Must be a value accepted by ValidateImageModel;
// empty falls back to DefaultImageModel.
ImageModel string
// Prompt is the text description of the image.
Prompt string
// AspectRatio is the desired aspect ratio, e.g. "16:9", "1:1", "9:16".
// Converted to a concrete pixel size by the provider implementation.
// Defaults to "1:1" if empty.
AspectRatio string
// OutputFormat is the desired image format: "png" (default), "jpg", "webp".
OutputFormat string
// RefImages contains the list of reference images.
RefImages []RefImage
}
// RefImage represents a single reference image for image-to-image or styling tasks.
type RefImage struct {
Data []byte
Base64 string
MimeType string
URL string
Strength float64
}
// NativeImageResult holds the result of a native image generation call.
type NativeImageResult struct {
// MimeType is the detected MIME type of the generated image (e.g. "image/png").
MimeType string
// Data is the raw decoded image bytes (NOT base64).
Data []byte
// Usage is optional token usage if the provider reports it.
Usage *Usage
}
// SizeFromAspect converts a common aspect ratio string to a pixel dimension
// string expected by image generation APIs (e.g. "1792x1024").
// Falls back to "1024x1024" for unrecognised ratios.
func SizeFromAspect(aspectRatio string) string {
switch aspectRatio {
case "16:9":
return "1792x1024"
case "9:16":
return "1024x1792"
case "3:4":
return "1024x1365"
case "4:3":
return "1365x1024"
default:
return "1024x1024"
}
}