From f6bb31c95657d219373ec09009e0f7bed991c439 Mon Sep 17 00:00:00 2001 From: seilk Date: Thu, 30 Apr 2026 00:55:45 +0900 Subject: [PATCH 1/3] fix(proxy): stringify tool_result images for OpenAI upstreams --- src/proxy/transformers/request-transformer.ts | 13 ++++- .../request-transformer-regressions.test.ts | 54 +++++++++++-------- 2 files changed, 45 insertions(+), 22 deletions(-) diff --git a/src/proxy/transformers/request-transformer.ts b/src/proxy/transformers/request-transformer.ts index b5877bd6..2f447fee 100644 --- a/src/proxy/transformers/request-transformer.ts +++ b/src/proxy/transformers/request-transformer.ts @@ -219,7 +219,18 @@ function convertToolResultContent(content: unknown, isError: boolean, label: str } if (parsed.type === 'image') { - throw new Error(`${label}[${index}].type "image" is not supported in tool_result content`); + const source = + typeof parsed.source === 'object' && parsed.source !== null + ? (parsed.source as Record) + : undefined; + const description = + source?.type === 'url' && typeof source.url === 'string' + ? source.url + : source?.type === 'base64' && typeof source.media_type === 'string' + ? `${source.media_type} base64 payload` + : 'unsupported image payload'; + parts.push(`[tool_result image omitted: ${description}]`); + continue; } if (typeof parsed.text === 'string') { diff --git a/tests/unit/proxy/transformers/request-transformer-regressions.test.ts b/tests/unit/proxy/transformers/request-transformer-regressions.test.ts index 9e4bc5a8..c34798bd 100644 --- a/tests/unit/proxy/transformers/request-transformer-regressions.test.ts +++ b/tests/unit/proxy/transformers/request-transformer-regressions.test.ts @@ -189,27 +189,39 @@ describe('ProxyRequestTransformer regressions', () => { ).toThrow('must start with tool_result blocks for pending tool_use ids'); }); - it('rejects tool_result content that cannot be represented as OpenAI tool text', () => { - expect(() => - new ProxyRequestTransformer().transform({ - messages: [ - { - role: 'assistant', - content: [{ type: 'tool_use', id: 'toolu_1', name: 'vision', input: { detail: 'high' } }], - }, - { - role: 'user', - content: [ - { - type: 'tool_result', - tool_use_id: 'toolu_1', - content: [{ type: 'image', source: { type: 'url', url: 'https://example.com/error.png' } }], - }, - ], - }, - ], - }) - ).toThrow('type "image" is not supported in tool_result content'); + it('converts tool_result image blocks to text placeholders for OpenAI-compatible tool messages', () => { + const result = new ProxyRequestTransformer().transform({ + messages: [ + { + role: 'assistant', + content: [{ type: 'tool_use', id: 'toolu_1', name: 'vision', input: { detail: 'high' } }], + }, + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'toolu_1', + content: [ + { type: 'text', text: 'screenshot captured' }, + { type: 'image', source: { type: 'url', url: 'https://example.com/error.png' } }, + { + type: 'image', + source: { type: 'base64', media_type: 'image/png', data: 'ZmFrZQ==' }, + }, + ], + }, + ], + }, + ], + }); + + expect(result.messages[1]).toEqual({ + role: 'tool', + tool_call_id: 'toolu_1', + content: + 'screenshot captured\n[tool_result image omitted: https://example.com/error.png]\n[tool_result image omitted: image/png base64 payload]', + }); }); it('rejects unsupported assistant blocks instead of silently dropping them', () => { From 83c16e216a2be549306434a1afa317844337b607 Mon Sep 17 00:00:00 2001 From: seilk Date: Thu, 30 Apr 2026 02:18:22 +0900 Subject: [PATCH 2/3] fix(proxy): avoid leaking tool_result image URLs --- src/proxy/transformers/request-transformer.ts | 4 ++-- .../transformers/request-transformer-regressions.test.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/proxy/transformers/request-transformer.ts b/src/proxy/transformers/request-transformer.ts index 2f447fee..0942244e 100644 --- a/src/proxy/transformers/request-transformer.ts +++ b/src/proxy/transformers/request-transformer.ts @@ -224,8 +224,8 @@ function convertToolResultContent(content: unknown, isError: boolean, label: str ? (parsed.source as Record) : undefined; const description = - source?.type === 'url' && typeof source.url === 'string' - ? source.url + source?.type === 'url' + ? 'url image payload' : source?.type === 'base64' && typeof source.media_type === 'string' ? `${source.media_type} base64 payload` : 'unsupported image payload'; diff --git a/tests/unit/proxy/transformers/request-transformer-regressions.test.ts b/tests/unit/proxy/transformers/request-transformer-regressions.test.ts index c34798bd..79ceba27 100644 --- a/tests/unit/proxy/transformers/request-transformer-regressions.test.ts +++ b/tests/unit/proxy/transformers/request-transformer-regressions.test.ts @@ -220,7 +220,7 @@ describe('ProxyRequestTransformer regressions', () => { role: 'tool', tool_call_id: 'toolu_1', content: - 'screenshot captured\n[tool_result image omitted: https://example.com/error.png]\n[tool_result image omitted: image/png base64 payload]', + 'screenshot captured\n[tool_result image omitted: url image payload]\n[tool_result image omitted: image/png base64 payload]', }); }); From 2c0f9c41ff1fff94c35e93535b9a8343ef7ce953 Mon Sep 17 00:00:00 2001 From: seilk Date: Thu, 30 Apr 2026 02:22:49 +0900 Subject: [PATCH 3/3] test(proxy): assert tool_result image URLs stay redacted --- .../transformers/request-transformer-regressions.test.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/unit/proxy/transformers/request-transformer-regressions.test.ts b/tests/unit/proxy/transformers/request-transformer-regressions.test.ts index 79ceba27..656c4e7a 100644 --- a/tests/unit/proxy/transformers/request-transformer-regressions.test.ts +++ b/tests/unit/proxy/transformers/request-transformer-regressions.test.ts @@ -204,7 +204,10 @@ describe('ProxyRequestTransformer regressions', () => { tool_use_id: 'toolu_1', content: [ { type: 'text', text: 'screenshot captured' }, - { type: 'image', source: { type: 'url', url: 'https://example.com/error.png' } }, + { + type: 'image', + source: { type: 'url', url: 'https://storage.example.com/error.png?X-Amz-Signature=secret' }, + }, { type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'ZmFrZQ==' }, @@ -222,6 +225,9 @@ describe('ProxyRequestTransformer regressions', () => { content: 'screenshot captured\n[tool_result image omitted: url image payload]\n[tool_result image omitted: image/png base64 payload]', }); + expect(result.messages[1]?.content).not.toContain('https://storage.example.com'); + expect(result.messages[1]?.content).not.toContain('X-Amz-Signature'); + expect(result.messages[1]?.content).not.toContain('secret'); }); it('rejects unsupported assistant blocks instead of silently dropping them', () => {