diff --git a/src/glmt/delta-accumulator.ts b/src/glmt/delta-accumulator.ts index 7923a0ae..7915267c 100644 --- a/src/glmt/delta-accumulator.ts +++ b/src/glmt/delta-accumulator.ts @@ -284,7 +284,10 @@ export class DeltaAccumulator { getToolCallBlockIndex(toolCallIndex: number): number { const toolCall = this.toolCallsIndex[toolCallIndex]; - return toolCall?.blockIndex ?? 0; + if (!toolCall || toolCall.blockIndex < 0) { + throw new Error(`Tool call ${toolCallIndex} does not have an assigned content block`); + } + return toolCall.blockIndex; } getUnstoppedBlocks(): ContentBlock[] { diff --git a/src/glmt/pipeline/tool-call-handler.ts b/src/glmt/pipeline/tool-call-handler.ts index 5bc7f46d..251813a4 100644 --- a/src/glmt/pipeline/tool-call-handler.ts +++ b/src/glmt/pipeline/tool-call-handler.ts @@ -5,20 +5,12 @@ * - Process tool call deltas from OpenAI * - Generate tool_use content blocks for Anthropic format * - Handle input_json_delta events - * - Close previous tool_use blocks before starting new ones (Anthropic sequential block requirement) */ import type { DeltaAccumulator } from '../delta-accumulator'; import type { OpenAIToolCallDelta, OpenAIToolCall, ContentBlock, AnthropicSSEEvent } from './types'; -import { ResponseBuilder } from './response-builder'; export class ToolCallHandler { - private responseBuilder: ResponseBuilder; - - constructor() { - this.responseBuilder = new ResponseBuilder(false); - } - processToolCalls(toolCalls: OpenAIToolCall[]): ContentBlock[] { const content: ContentBlock[] = []; @@ -54,12 +46,8 @@ export class ToolCallHandler { accumulator.addToolCallDelta(toolCallDelta); if (isNewToolCall) { - const previousBlock = accumulator.getCurrentBlock(); - if (previousBlock && previousBlock.type === 'tool_use' && !previousBlock.stopped) { - events.push(this.responseBuilder.createContentBlockStopEvent(previousBlock)); - previousBlock.stopped = true; - } - + // OpenAI may interleave tool_call fragments across chunks, so blocks must stay open + // until the stream finalizes. Closing on a later index truncates earlier tool input. const block = accumulator.startBlock('tool_use'); const toolCall = accumulator.getToolCall(toolCallDelta.index); accumulator.setToolCallBlockIndex(toolCallDelta.index, block.index); diff --git a/src/proxy/transformers/request-transformer.ts b/src/proxy/transformers/request-transformer.ts index a9c6ab69..a8ccb98c 100644 --- a/src/proxy/transformers/request-transformer.ts +++ b/src/proxy/transformers/request-transformer.ts @@ -457,9 +457,35 @@ function transformMessages(messagesValue: unknown): OpenAIMessage[] { } if (role === 'user') { - const preToolResultParts: OpenAIContentPart[] = []; + const userParts: OpenAIContentPart[] = []; + const followUpParts: OpenAIContentPart[] = []; const resolvedToolUseIds = new Set(); + const handleUserPart = ( + part: OpenAIContentPart, + blockIndex: number, + kind: 'text' | 'image' + ) => { + if (!pendingToolUseIds || pendingToolUseIds.size === 0) { + userParts.push(part); + return; + } + + if (resolvedToolUseIds.size === 0) { + throw new Error( + `messages[${messageIndex}].content[${blockIndex}] ${kind} is not allowed before tool_result blocks for pending tool_use ids` + ); + } + + if (resolvedToolUseIds.size !== pendingToolUseIds.size) { + throw new Error( + `messages[${messageIndex}].content[${blockIndex}] ${kind} is not allowed between tool_result blocks for pending tool_use ids` + ); + } + + followUpParts.push(part); + }; + content.forEach((block, blockIndex) => { const parsed = assertObject( block, @@ -472,25 +498,16 @@ function transformMessages(messagesValue: unknown): OpenAIMessage[] { if (parsed.type === 'text') { const text = typeof parsed.text === 'string' ? parsed.text : ''; - if (pendingToolUseIds && pendingToolUseIds.size > 0 && !resolvedToolUseIds.size) { - preToolResultParts.push({ type: 'text', text }); - } else { - translatedMessages.push({ role: 'user', content: text }); - } + handleUserPart({ type: 'text', text }, blockIndex, 'text'); return; } if (isImageBlock(parsed)) { - if (pendingToolUseIds && pendingToolUseIds.size > 0 && !resolvedToolUseIds.size) { - preToolResultParts.push( - toImagePart(parsed, `messages[${messageIndex}].content[${blockIndex}]`) - ); - } else { - translatedMessages.push({ - role: 'user', - content: [toImagePart(parsed, `messages[${messageIndex}].content[${blockIndex}]`)], - }); - } + handleUserPart( + toImagePart(parsed, `messages[${messageIndex}].content[${blockIndex}]`), + blockIndex, + 'image' + ); return; } @@ -555,8 +572,12 @@ function transformMessages(messagesValue: unknown): OpenAIMessage[] { ); } - if (preToolResultParts.length > 0) { - flushUserContent(translatedMessages, preToolResultParts); + if (userParts.length > 0) { + flushUserContent(translatedMessages, userParts); + } + + if (followUpParts.length > 0) { + flushUserContent(translatedMessages, followUpParts); } return; } diff --git a/tests/integration/proxy/messages-endpoint.test.ts b/tests/integration/proxy/messages-endpoint.test.ts index cf1d408b..90c819eb 100644 --- a/tests/integration/proxy/messages-endpoint.test.ts +++ b/tests/integration/proxy/messages-endpoint.test.ts @@ -23,10 +23,34 @@ function startMockUpstream(): Promise { body += chunk.toString(); } upstreamBody = JSON.parse(body); - const parsed = upstreamBody as { stream?: boolean }; + const parsed = upstreamBody as { + stream?: boolean; + messages?: Array<{ role?: string; content?: string | Array<{ type?: string; text?: string }> }>; + }; if (parsed.stream) { res.writeHead(200, { 'Content-Type': 'text/event-stream' }); + + if (parsed.messages?.[0]?.content === 'interleaved tool fragments') { + res.write( + 'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"search"}}]}}]}\n\n' + ); + res.write( + 'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"id":"call_2","type":"function","function":{"name":"open"}}]}}]}\n\n' + ); + res.write( + 'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\\"path\\":\\"a.ts\\"}"}}]}}]}\n\n' + ); + res.write( + 'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\\"path\\":\\"b.ts\\"}"}}]}}]}\n\n' + ); + res.write( + 'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":9,"completion_tokens":4}}\n\n' + ); + res.end('data: [DONE]\n\n'); + return; + } + res.write( 'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello"}}]}\n\n' ); @@ -356,6 +380,66 @@ describe('openai proxy messages endpoint', () => { expect(userAfterTool?.[1]?.content).toBe('What should I do next?'); }); + it('rejects text before tool_result blocks when tool results are pending', async () => { + const response = await requestProxy({ + model: 'hf-model', + stream: false, + messages: [ + { role: 'user', content: 'search docs' }, + { + role: 'assistant', + content: [{ type: 'tool_use', id: 'toolu_01', name: 'search', input: { q: 'docs' } }], + }, + { + role: 'user', + content: [ + { type: 'text', text: 'before' }, + { type: 'tool_result', tool_use_id: 'toolu_01', content: 'found 3 docs' }, + ], + }, + ], + }); + + const body = (await response.json()) as { error?: { type?: string; message?: string } }; + expect(response.status).toBe(400); + expect(body.error?.type).toBe('invalid_request_error'); + expect(body.error?.message).toContain( + 'text is not allowed before tool_result blocks for pending tool_use ids' + ); + }); + + it('rejects follow-up text between pending tool_result blocks', async () => { + const response = await requestProxy({ + model: 'hf-model', + stream: false, + messages: [ + { role: 'user', content: 'read both files' }, + { + role: 'assistant', + content: [ + { type: 'tool_use', id: 'toolu_01', name: 'Read', input: { file_path: 'a.ts' } }, + { type: 'tool_use', id: 'toolu_02', name: 'Read', input: { file_path: 'b.ts' } }, + ], + }, + { + role: 'user', + content: [ + { type: 'tool_result', tool_use_id: 'toolu_01', content: 'content of a' }, + { type: 'text', text: 'Now compare them' }, + { type: 'tool_result', tool_use_id: 'toolu_02', content: 'content of b' }, + ], + }, + ], + }); + + const body = (await response.json()) as { error?: { type?: string; message?: string } }; + expect(response.status).toBe(400); + expect(body.error?.type).toBe('invalid_request_error'); + expect(body.error?.message).toContain( + 'text is not allowed between tool_result blocks for pending tool_use ids' + ); + }); + it('translates parallel tool calls with streaming', async () => { const response = await requestProxy({ model: 'hf-model', @@ -409,4 +493,36 @@ describe('openai proxy messages endpoint', () => { expect(toolMsgs?.[0]?.tool_call_id).toBe('toolu_01'); expect(toolMsgs?.[1]?.tool_call_id).toBe('toolu_02'); }); + + it('streams interleaved tool call fragments without premature block stops', async () => { + const response = await requestProxy({ + model: 'hf-model', + stream: true, + messages: [{ role: 'user', content: [{ type: 'text', text: 'interleaved tool fragments' }] }], + tools: [ + { + name: 'search', + description: 'Search docs', + input_schema: { type: 'object', properties: { path: { type: 'string' } } }, + }, + { + name: 'open', + description: 'Open a file', + input_schema: { type: 'object', properties: { path: { type: 'string' } } }, + }, + ], + }); + + const body = await response.text(); + expect(response.status).toBe(200); + expect(body.match(/event: content_block_start/g)?.length).toBe(2); + expect(body.match(/event: content_block_stop/g)?.length).toBe(2); + + const stopIndex = body.indexOf('event: content_block_stop'); + const deltaAIndex = body.indexOf('"partial_json":"{\\"path\\":\\"a.ts\\"}"'); + const deltaBIndex = body.indexOf('"partial_json":"{\\"path\\":\\"b.ts\\"}"'); + expect(deltaAIndex).toBeGreaterThan(-1); + expect(deltaBIndex).toBeGreaterThan(-1); + expect(stopIndex).toBeGreaterThan(deltaBIndex); + }); }); diff --git a/tests/unit/proxy/transformers/request-transformer-regressions.test.ts b/tests/unit/proxy/transformers/request-transformer-regressions.test.ts index af83395d..77b56294 100644 --- a/tests/unit/proxy/transformers/request-transformer-regressions.test.ts +++ b/tests/unit/proxy/transformers/request-transformer-regressions.test.ts @@ -120,8 +120,48 @@ describe('ProxyRequestTransformer regressions', () => { }, ], }) + ).toThrow('text is not allowed before tool_result blocks for pending tool_use ids'); + + expect(() => + new ProxyRequestTransformer().transform({ + messages: [ + { + role: 'assistant', + content: [{ type: 'tool_use', id: 'toolu_1', name: 'vision', input: { detail: 'high' } }], + }, + { + role: 'user', + content: [ + { type: 'tool_result', tool_use_id: 'toolu_1', content: 'result' }, + { type: 'text', text: 'follow-up' }, + ], + }, + ], + }) ).not.toThrow(); + expect(() => + new ProxyRequestTransformer().transform({ + messages: [ + { + role: 'assistant', + content: [ + { type: 'tool_use', id: 'toolu_1', name: 'search', input: { q: 'docs' } }, + { type: 'tool_use', id: 'toolu_2', name: 'open', input: { url: 'https://example.com' } }, + ], + }, + { + role: 'user', + content: [ + { type: 'tool_result', tool_use_id: 'toolu_1', content: 'partial' }, + { type: 'text', text: 'follow-up' }, + { type: 'tool_result', tool_use_id: 'toolu_2', content: 'done' }, + ], + }, + ], + }) + ).toThrow('text is not allowed between tool_result blocks for pending tool_use ids'); + expect(() => new ProxyRequestTransformer().transform({ messages: [