Files
goclaw/internal/providers/codex_native_image_test.go
T
thotam a5a853f461 feat(tools): image reference processing and native provider support (#1251)
* feat(tools): implement image reference processing and native provider support

- Support OpenAI image edits via both Multipart form-data and JSON payloads
- Automatically append reference image descriptions to prompt under [Reference Image Roles]
- Support downloading image URLs for Gemini native image generation
- Deduplicate reference images to optimize API request size
- Add unit tests for Codex, DashScope, MiniMax, BytePlus and local/remote path resolution

* fix(tools): SSRF-guard reference-image URL downloads in create_image

downloadImageBytes fetched caller-supplied ref_images[].url with a plain
http.Client and unbounded io.ReadAll — no SSRF validation, redirect
policy, or size cap, letting the gateway dial loopback/private/metadata
hosts or read arbitrarily large responses.

- Validate the URL via security.Validate and pin the resolved IP, then
  download through security.NewSafeClient (pinned dial, no redirects).
- Cap the response with a bounded read (refImageMaxBytes, 20 MB).
- Reject non-HTTP(S) reference URLs up front (file://, data:, gopher://)
  so provider-forwarded URLs stay HTTP(S)-only; document the trust
  boundary between gateway-side fetch and provider-forwarded URLs.
- Add regression tests: blocked loopback/private/metadata, unfollowed
  redirect, oversized response, and non-http(s) scheme rejection.
2026-06-21 18:27:51 +07:00

484 lines
16 KiB
Go

package providers
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
// minimalPNGForProviders is a 1x1 transparent PNG in base64 used by native image tests.
const minimalPNGForProviders = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
// mockImageServer returns a test server that captures request bodies and returns a
// minimal successful image generation response. The captured pointer is written on
// each request.
func mockImageServer(t *testing.T, captured *[]byte) *httptest.Server {
t.Helper()
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(r.Body)
if err != nil {
t.Errorf("read body: %v", err)
http.Error(w, "read error", http.StatusInternalServerError)
return
}
*captured = body
resp := map[string]any{
"id": "resp_test",
"status": "completed",
"output": []map[string]any{
{
"type": "image_generation_call",
"result": minimalPNGForProviders,
"output_format": "png",
},
},
}
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(resp); err != nil {
t.Errorf("encode response: %v", err)
}
}))
}
// TestCodexGenerateImage_BuildsNativeRequest verifies that GenerateImage sends the
// correct JSON body to the Responses API: model, stream:false, input, tools, and
// tool_choice. The test captures the raw request body from a mock server and
// asserts each required field is present and well-formed.
//
// Sub-cases:
// - Default (empty ImageModel) → tools[0].model == "gpt-image-2"
// - Legacy (ImageModel: "gpt-image-1.5") → tools[0].model == "gpt-image-1.5"
// - Rejected (ImageModel: "dall-e-3") → GenerateImage returns error containing "unsupported image model"
func TestCodexGenerateImage_BuildsNativeRequest(t *testing.T) {
var captured []byte
server := mockImageServer(t, &captured)
defer server.Close()
p := NewCodexProvider("codex-test", &staticTokenSource{token: "tok"}, server.URL, "gpt-image-2")
p.retryConfig.Attempts = 1
req := NativeImageRequest{
Model: "gpt-image-2",
Prompt: "A red circle on a white background",
AspectRatio: "16:9",
OutputFormat: "png",
}
result, err := p.GenerateImage(context.Background(), req)
if err != nil {
t.Fatalf("GenerateImage returned error: %v", err)
}
if len(result.Data) == 0 {
t.Fatal("GenerateImage returned empty Data")
}
// Verify outbound request body shape.
var body map[string]any
if err := json.Unmarshal(captured, &body); err != nil {
t.Fatalf("unmarshal captured body: %v", err)
}
// model field (outer Responses API model, not image model)
if model, _ := body["model"].(string); model != "gpt-image-2" {
t.Errorf("body[model] = %q, want %q", model, "gpt-image-2")
}
// Responses API requires stream:true — non-streaming requests are rejected with
// HTTP 400 "Stream must be set to true". Final image is assembled from SSE events.
if stream, _ := body["stream"].(bool); !stream {
t.Error("body[stream] must be true (Responses API rejects stream:false)")
}
// instructions is required by Responses API — must be non-empty.
if instr, _ := body["instructions"].(string); instr == "" {
t.Error("body[instructions] must be non-empty (Responses API rejects requests without instructions)")
}
// input must be an array with one user message
inputs, ok := body["input"].([]any)
if !ok || len(inputs) != 1 {
t.Fatalf("body[input]: expected []any length 1, got %T len %d", body["input"], len(inputs))
}
userMsg, ok := inputs[0].(map[string]any)
if !ok {
t.Fatalf("input[0] is not a map: %T", inputs[0])
}
if role, _ := userMsg["role"].(string); role != "user" {
t.Errorf("input[0].role = %q, want %q", role, "user")
}
contents, ok := userMsg["content"].([]any)
if !ok || len(contents) != 1 {
t.Fatalf("input[0].content: expected []any length 1, got %T len %d", userMsg["content"], len(contents))
}
contentPart, ok := contents[0].(map[string]any)
if !ok {
t.Fatalf("content[0] is not a map: %T", contents[0])
}
if typ, _ := contentPart["type"].(string); typ != "input_text" {
t.Errorf("content[0].type = %q, want %q", typ, "input_text")
}
if text, _ := contentPart["text"].(string); text != req.Prompt {
t.Errorf("content[0].text = %q, want %q", text, req.Prompt)
}
// tools must contain one image_generation entry
tools, ok := body["tools"].([]any)
if !ok || len(tools) != 1 {
t.Fatalf("body[tools]: expected []any length 1, got %T len %d", body["tools"], len(tools))
}
tool, ok := tools[0].(map[string]any)
if !ok {
t.Fatalf("tools[0] is not a map: %T", tools[0])
}
if typ, _ := tool["type"].(string); typ != "image_generation" {
t.Errorf("tools[0].type = %q, want %q", typ, "image_generation")
}
// size should map to 1792x1024 for 16:9
wantSize := SizeFromAspect("16:9")
if size, _ := tool["size"].(string); size != wantSize {
t.Errorf("tools[0].size = %q, want %q", size, wantSize)
}
if fmt.Sprint(tool["output_format"]) != "png" {
t.Errorf("tools[0].output_format = %v, want png", tool["output_format"])
}
// tools[0].model must be gpt-image-2 (default when ImageModel is empty)
if imgModel, _ := tool["model"].(string); imgModel != DefaultImageModel {
t.Errorf("tools[0].model = %q, want %q (default)", imgModel, DefaultImageModel)
}
// tool_choice must force image_generation
toolChoice, ok := body["tool_choice"].(map[string]any)
if !ok {
t.Fatalf("body[tool_choice] is not a map: %T", body["tool_choice"])
}
if typ, _ := toolChoice["type"].(string); typ != "image_generation" {
t.Errorf("tool_choice.type = %q, want %q", typ, "image_generation")
}
}
// TestCodexGenerateImage_ImageModelDefault verifies that an empty ImageModel results
// in the default gpt-image-2 model in the outbound tools[0].model field.
func TestCodexGenerateImage_ImageModelDefault(t *testing.T) {
var captured []byte
server := mockImageServer(t, &captured)
defer server.Close()
p := NewCodexProvider("codex-test", &staticTokenSource{token: "tok"}, server.URL, "gpt-image-2")
p.retryConfig.Attempts = 1
_, err := p.GenerateImage(context.Background(), NativeImageRequest{
Prompt: "test",
ImageModel: "", // explicitly empty — should default to gpt-image-2
AspectRatio: "1:1",
})
if err != nil {
t.Fatalf("GenerateImage returned error: %v", err)
}
var body map[string]any
if err := json.Unmarshal(captured, &body); err != nil {
t.Fatalf("unmarshal captured body: %v", err)
}
tools, _ := body["tools"].([]any)
if len(tools) == 0 {
t.Fatal("tools array is empty")
}
tool, _ := tools[0].(map[string]any)
if imgModel, _ := tool["model"].(string); imgModel != "gpt-image-2" {
t.Errorf("tools[0].model = %q, want gpt-image-2 (default)", imgModel)
}
}
// TestCodexGenerateImage_ImageModelLegacy verifies that ImageModel "gpt-image-1.5"
// is forwarded to the outbound tools[0].model field.
func TestCodexGenerateImage_ImageModelLegacy(t *testing.T) {
var captured []byte
server := mockImageServer(t, &captured)
defer server.Close()
p := NewCodexProvider("codex-test", &staticTokenSource{token: "tok"}, server.URL, "gpt-image-2")
p.retryConfig.Attempts = 1
_, err := p.GenerateImage(context.Background(), NativeImageRequest{
Prompt: "test",
ImageModel: "gpt-image-1.5",
AspectRatio: "1:1",
})
if err != nil {
t.Fatalf("GenerateImage returned error: %v", err)
}
var body map[string]any
if err := json.Unmarshal(captured, &body); err != nil {
t.Fatalf("unmarshal captured body: %v", err)
}
tools, _ := body["tools"].([]any)
if len(tools) == 0 {
t.Fatal("tools array is empty")
}
tool, _ := tools[0].(map[string]any)
if imgModel, _ := tool["model"].(string); imgModel != "gpt-image-1.5" {
t.Errorf("tools[0].model = %q, want gpt-image-1.5 (legacy)", imgModel)
}
}
// TestCodexGenerateImage_ImageModelRejected verifies that an unsupported image model
// causes GenerateImage to return an error containing "unsupported image model" before
// making any HTTP request.
func TestCodexGenerateImage_ImageModelRejected(t *testing.T) {
requestMade := false
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requestMade = true
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
p := NewCodexProvider("codex-test", &staticTokenSource{token: "tok"}, server.URL, "gpt-image-2")
p.retryConfig.Attempts = 1
_, err := p.GenerateImage(context.Background(), NativeImageRequest{
Prompt: "test",
ImageModel: "dall-e-3",
})
if err == nil {
t.Fatal("expected error for unsupported image model, got nil")
}
if !strings.Contains(err.Error(), "unsupported image model") {
t.Errorf("error %q does not contain 'unsupported image model'", err.Error())
}
if requestMade {
t.Error("HTTP request was made despite invalid image model (should have been rejected before the request)")
}
}
// TestCodexGenerateImage_SSEFallback verifies that GenerateImage correctly parses
// an SSE-format response when the server returns streamed lines instead of a JSON blob.
func TestCodexGenerateImage_SSEFallback(t *testing.T) {
imgB64 := minimalPNGForProviders
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/event-stream")
// Emit a response.completed SSE event with an image_generation_call.
ev := codexSSEEvent{
Type: "response.completed",
Response: &codexAPIResponse{
ID: "resp_sse",
Status: "completed",
Output: []codexItem{
{
ID: "ig_1",
Type: "image_generation_call",
OutputFormat: "png",
Result: imgB64,
},
},
Usage: &codexUsage{InputTokens: 5, OutputTokens: 5, TotalTokens: 10},
},
}
b, _ := json.Marshal(ev)
fmt.Fprintf(w, "data: %s\n\n", b)
fmt.Fprint(w, "data: [DONE]\n\n")
}))
defer server.Close()
p := NewCodexProvider("codex-test", &staticTokenSource{token: "tok"}, server.URL, "gpt-image-2")
p.retryConfig.Attempts = 1
result, err := p.GenerateImage(context.Background(), NativeImageRequest{
Prompt: "A blue square",
OutputFormat: "png",
})
if err != nil {
t.Fatalf("GenerateImage SSE fallback: %v", err)
}
if result.MimeType != "image/png" {
t.Errorf("MimeType = %q, want image/png", result.MimeType)
}
want, _ := base64.StdEncoding.DecodeString(imgB64)
if len(result.Data) != len(want) {
t.Errorf("Data length = %d, want %d", len(result.Data), len(want))
}
if result.Usage == nil {
t.Error("Usage is nil")
} else if result.Usage.TotalTokens != 10 {
t.Errorf("Usage.TotalTokens = %d, want 10", result.Usage.TotalTokens)
}
}
// TestCodexGenerateImage_NoPrompt verifies that an empty prompt returns an error
// before making any HTTP request.
func TestCodexGenerateImage_NoPrompt(t *testing.T) {
p := NewCodexProvider("codex-test", &staticTokenSource{token: "tok"}, "http://localhost", "gpt-image-2")
p.retryConfig.Attempts = 1
_, err := p.GenerateImage(context.Background(), NativeImageRequest{Prompt: ""})
if err == nil {
t.Fatal("expected error for empty prompt, got nil")
}
}
// TestSizeFromAspect verifies the aspect ratio → pixel dimension mapping.
func TestSizeFromAspect(t *testing.T) {
cases := []struct {
ratio string
want string
}{
{"1:1", "1024x1024"},
{"16:9", "1792x1024"},
{"9:16", "1024x1792"},
{"4:3", "1365x1024"},
{"3:4", "1024x1365"},
{"", "1024x1024"},
{"custom", "1024x1024"},
}
for _, tc := range cases {
got := SizeFromAspect(tc.ratio)
if got != tc.want {
t.Errorf("SizeFromAspect(%q) = %q, want %q", tc.ratio, got, tc.want)
}
}
}
// TestCodexGenerateImage_WithReferenceImage verifies that passing a single RefImage
// embeds the image in the input content and does not populate input_reference in tools[0].
func TestCodexGenerateImage_WithReferenceImage(t *testing.T) {
var captured []byte
server := mockImageServer(t, &captured)
defer server.Close()
p := NewCodexProvider("codex-test", &staticTokenSource{token: "tok"}, server.URL, "gpt-image-2")
p.retryConfig.Attempts = 1
req := NativeImageRequest{
Model: "gpt-image-2",
Prompt: "A red circle",
RefImages: []RefImage{
{
URL: "https://example.com/ref.png",
},
},
AspectRatio: "1:1",
OutputFormat: "png",
}
_, err := p.GenerateImage(context.Background(), req)
if err != nil {
t.Fatalf("GenerateImage returned error: %v", err)
}
var body map[string]any
if err := json.Unmarshal(captured, &body); err != nil {
t.Fatalf("unmarshal captured body: %v", err)
}
// 1. Verify input contains input_image and input_text in user content
inputs, ok := body["input"].([]any)
if !ok || len(inputs) != 1 {
t.Fatalf("body[input]: expected []any length 1, got %T len %d", body["input"], len(inputs))
}
userMsg, ok := inputs[0].(map[string]any)
if !ok {
t.Fatalf("inputs[0] is not a map")
}
contents, ok := userMsg["content"].([]any)
if !ok || len(contents) != 2 {
t.Fatalf("content: expected []any length 2, got %T len %d", userMsg["content"], len(contents))
}
imgPart, ok := contents[0].(map[string]any)
if !ok {
t.Fatalf("contents[0] is not a map")
}
if imgPart["type"] != "input_image" || imgPart["image_url"] != "https://example.com/ref.png" {
t.Errorf("expected input_image with url, got: %v", imgPart)
}
textPart, ok := contents[1].(map[string]any)
if !ok {
t.Fatalf("contents[1] is not a map")
}
if textPart["type"] != "input_text" || textPart["text"] != "A red circle" {
t.Errorf("expected input_text with prompt, got: %v", textPart)
}
// 2. Verify tools[0] does not contain input_reference
tools, ok := body["tools"].([]any)
if !ok || len(tools) != 1 {
t.Fatalf("tools shape invalid")
}
tool, ok := tools[0].(map[string]any)
if !ok {
t.Fatalf("tools[0] not map")
}
if _, has := tool["input_reference"]; has {
t.Error("image_generation tool must not contain 'input_reference' field")
}
}
// TestCodexGenerateImage_WithMultipleReferenceImages verifies that passing multiple RefImages
// embeds all of them in the input content in the correct order.
func TestCodexGenerateImage_WithMultipleReferenceImages(t *testing.T) {
var captured []byte
server := mockImageServer(t, &captured)
defer server.Close()
p := NewCodexProvider("codex-test", &staticTokenSource{token: "tok"}, server.URL, "gpt-image-2")
p.retryConfig.Attempts = 1
req := NativeImageRequest{
Model: "gpt-image-2",
Prompt: "A red circle",
RefImages: []RefImage{
{
URL: "https://example.com/ref1.png",
},
{
Base64: "b64data",
MimeType: "image/jpeg",
},
},
AspectRatio: "1:1",
OutputFormat: "png",
}
_, err := p.GenerateImage(context.Background(), req)
if err != nil {
t.Fatalf("GenerateImage returned error: %v", err)
}
var body map[string]any
if err := json.Unmarshal(captured, &body); err != nil {
t.Fatalf("unmarshal captured body: %v", err)
}
inputs, _ := body["input"].([]any)
userMsg, _ := inputs[0].(map[string]any)
contents, _ := userMsg["content"].([]any)
// Expected: 2 images + 1 text prompt = 3 content parts
if len(contents) != 3 {
t.Fatalf("content: expected []any length 3, got len %d", len(contents))
}
imgPart1, _ := contents[0].(map[string]any)
if imgPart1["type"] != "input_image" || imgPart1["image_url"] != "https://example.com/ref1.png" {
t.Errorf("expected first input_image with url, got: %v", imgPart1)
}
imgPart2, _ := contents[1].(map[string]any)
if imgPart2["type"] != "input_image" || imgPart2["image_url"] != "data:image/jpeg;base64,b64data" {
t.Errorf("expected second input_image with base64 data url, got: %v", imgPart2)
}
textPart, _ := contents[2].(map[string]any)
if textPart["type"] != "input_text" || textPart["text"] != "A red circle" {
t.Errorf("expected input_text with prompt, got: %v", textPart)
}
}