fix(transformers): preserve tool ordering across proxy streams

- reject pending tool_result layouts that cannot be translated without reordering user content

- keep interleaved GLMT tool_use blocks open until finalization instead of stopping early

- cover leading/interleaved tool_result regressions and interleaved streaming tool fragments
This commit is contained in:
Tam Nhu Tran
2026-04-20 13:17:30 -04:00
parent 889d3d89e9
commit 399f403322
5 changed files with 202 additions and 34 deletions
+4 -1
View File
@@ -284,7 +284,10 @@ export class DeltaAccumulator {
getToolCallBlockIndex(toolCallIndex: number): number {
const toolCall = this.toolCallsIndex[toolCallIndex];
return toolCall?.blockIndex ?? 0;
if (!toolCall || toolCall.blockIndex < 0) {
throw new Error(`Tool call ${toolCallIndex} does not have an assigned content block`);
}
return toolCall.blockIndex;
}
getUnstoppedBlocks(): ContentBlock[] {
+2 -14
View File
@@ -5,20 +5,12 @@
* - Process tool call deltas from OpenAI
* - Generate tool_use content blocks for Anthropic format
* - Handle input_json_delta events
* - Close previous tool_use blocks before starting new ones (Anthropic sequential block requirement)
*/
import type { DeltaAccumulator } from '../delta-accumulator';
import type { OpenAIToolCallDelta, OpenAIToolCall, ContentBlock, AnthropicSSEEvent } from './types';
import { ResponseBuilder } from './response-builder';
export class ToolCallHandler {
private responseBuilder: ResponseBuilder;
constructor() {
this.responseBuilder = new ResponseBuilder(false);
}
processToolCalls(toolCalls: OpenAIToolCall[]): ContentBlock[] {
const content: ContentBlock[] = [];
@@ -54,12 +46,8 @@ export class ToolCallHandler {
accumulator.addToolCallDelta(toolCallDelta);
if (isNewToolCall) {
const previousBlock = accumulator.getCurrentBlock();
if (previousBlock && previousBlock.type === 'tool_use' && !previousBlock.stopped) {
events.push(this.responseBuilder.createContentBlockStopEvent(previousBlock));
previousBlock.stopped = true;
}
// OpenAI may interleave tool_call fragments across chunks, so blocks must stay open
// until the stream finalizes. Closing on a later index truncates earlier tool input.
const block = accumulator.startBlock('tool_use');
const toolCall = accumulator.getToolCall(toolCallDelta.index);
accumulator.setToolCallBlockIndex(toolCallDelta.index, block.index);
+39 -18
View File
@@ -457,9 +457,35 @@ function transformMessages(messagesValue: unknown): OpenAIMessage[] {
}
if (role === 'user') {
const preToolResultParts: OpenAIContentPart[] = [];
const userParts: OpenAIContentPart[] = [];
const followUpParts: OpenAIContentPart[] = [];
const resolvedToolUseIds = new Set<string>();
const handleUserPart = (
part: OpenAIContentPart,
blockIndex: number,
kind: 'text' | 'image'
) => {
if (!pendingToolUseIds || pendingToolUseIds.size === 0) {
userParts.push(part);
return;
}
if (resolvedToolUseIds.size === 0) {
throw new Error(
`messages[${messageIndex}].content[${blockIndex}] ${kind} is not allowed before tool_result blocks for pending tool_use ids`
);
}
if (resolvedToolUseIds.size !== pendingToolUseIds.size) {
throw new Error(
`messages[${messageIndex}].content[${blockIndex}] ${kind} is not allowed between tool_result blocks for pending tool_use ids`
);
}
followUpParts.push(part);
};
content.forEach((block, blockIndex) => {
const parsed = assertObject(
block,
@@ -472,25 +498,16 @@ function transformMessages(messagesValue: unknown): OpenAIMessage[] {
if (parsed.type === 'text') {
const text = typeof parsed.text === 'string' ? parsed.text : '';
if (pendingToolUseIds && pendingToolUseIds.size > 0 && !resolvedToolUseIds.size) {
preToolResultParts.push({ type: 'text', text });
} else {
translatedMessages.push({ role: 'user', content: text });
}
handleUserPart({ type: 'text', text }, blockIndex, 'text');
return;
}
if (isImageBlock(parsed)) {
if (pendingToolUseIds && pendingToolUseIds.size > 0 && !resolvedToolUseIds.size) {
preToolResultParts.push(
toImagePart(parsed, `messages[${messageIndex}].content[${blockIndex}]`)
);
} else {
translatedMessages.push({
role: 'user',
content: [toImagePart(parsed, `messages[${messageIndex}].content[${blockIndex}]`)],
});
}
handleUserPart(
toImagePart(parsed, `messages[${messageIndex}].content[${blockIndex}]`),
blockIndex,
'image'
);
return;
}
@@ -555,8 +572,12 @@ function transformMessages(messagesValue: unknown): OpenAIMessage[] {
);
}
if (preToolResultParts.length > 0) {
flushUserContent(translatedMessages, preToolResultParts);
if (userParts.length > 0) {
flushUserContent(translatedMessages, userParts);
}
if (followUpParts.length > 0) {
flushUserContent(translatedMessages, followUpParts);
}
return;
}
@@ -23,10 +23,34 @@ function startMockUpstream(): Promise<void> {
body += chunk.toString();
}
upstreamBody = JSON.parse(body);
const parsed = upstreamBody as { stream?: boolean };
const parsed = upstreamBody as {
stream?: boolean;
messages?: Array<{ role?: string; content?: string | Array<{ type?: string; text?: string }> }>;
};
if (parsed.stream) {
res.writeHead(200, { 'Content-Type': 'text/event-stream' });
if (parsed.messages?.[0]?.content === 'interleaved tool fragments') {
res.write(
'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"search"}}]}}]}\n\n'
);
res.write(
'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"id":"call_2","type":"function","function":{"name":"open"}}]}}]}\n\n'
);
res.write(
'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\\"path\\":\\"a.ts\\"}"}}]}}]}\n\n'
);
res.write(
'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"tool_calls":[{"index":1,"function":{"arguments":"{\\"path\\":\\"b.ts\\"}"}}]}}]}\n\n'
);
res.write(
'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":9,"completion_tokens":4}}\n\n'
);
res.end('data: [DONE]\n\n');
return;
}
res.write(
'data: {"id":"chatcmpl_1","model":"hf-model","choices":[{"index":0,"delta":{"role":"assistant","content":"Hello"}}]}\n\n'
);
@@ -356,6 +380,66 @@ describe('openai proxy messages endpoint', () => {
expect(userAfterTool?.[1]?.content).toBe('What should I do next?');
});
it('rejects text before tool_result blocks when tool results are pending', async () => {
const response = await requestProxy({
model: 'hf-model',
stream: false,
messages: [
{ role: 'user', content: 'search docs' },
{
role: 'assistant',
content: [{ type: 'tool_use', id: 'toolu_01', name: 'search', input: { q: 'docs' } }],
},
{
role: 'user',
content: [
{ type: 'text', text: 'before' },
{ type: 'tool_result', tool_use_id: 'toolu_01', content: 'found 3 docs' },
],
},
],
});
const body = (await response.json()) as { error?: { type?: string; message?: string } };
expect(response.status).toBe(400);
expect(body.error?.type).toBe('invalid_request_error');
expect(body.error?.message).toContain(
'text is not allowed before tool_result blocks for pending tool_use ids'
);
});
it('rejects follow-up text between pending tool_result blocks', async () => {
const response = await requestProxy({
model: 'hf-model',
stream: false,
messages: [
{ role: 'user', content: 'read both files' },
{
role: 'assistant',
content: [
{ type: 'tool_use', id: 'toolu_01', name: 'Read', input: { file_path: 'a.ts' } },
{ type: 'tool_use', id: 'toolu_02', name: 'Read', input: { file_path: 'b.ts' } },
],
},
{
role: 'user',
content: [
{ type: 'tool_result', tool_use_id: 'toolu_01', content: 'content of a' },
{ type: 'text', text: 'Now compare them' },
{ type: 'tool_result', tool_use_id: 'toolu_02', content: 'content of b' },
],
},
],
});
const body = (await response.json()) as { error?: { type?: string; message?: string } };
expect(response.status).toBe(400);
expect(body.error?.type).toBe('invalid_request_error');
expect(body.error?.message).toContain(
'text is not allowed between tool_result blocks for pending tool_use ids'
);
});
it('translates parallel tool calls with streaming', async () => {
const response = await requestProxy({
model: 'hf-model',
@@ -409,4 +493,36 @@ describe('openai proxy messages endpoint', () => {
expect(toolMsgs?.[0]?.tool_call_id).toBe('toolu_01');
expect(toolMsgs?.[1]?.tool_call_id).toBe('toolu_02');
});
it('streams interleaved tool call fragments without premature block stops', async () => {
const response = await requestProxy({
model: 'hf-model',
stream: true,
messages: [{ role: 'user', content: [{ type: 'text', text: 'interleaved tool fragments' }] }],
tools: [
{
name: 'search',
description: 'Search docs',
input_schema: { type: 'object', properties: { path: { type: 'string' } } },
},
{
name: 'open',
description: 'Open a file',
input_schema: { type: 'object', properties: { path: { type: 'string' } } },
},
],
});
const body = await response.text();
expect(response.status).toBe(200);
expect(body.match(/event: content_block_start/g)?.length).toBe(2);
expect(body.match(/event: content_block_stop/g)?.length).toBe(2);
const stopIndex = body.indexOf('event: content_block_stop');
const deltaAIndex = body.indexOf('"partial_json":"{\\"path\\":\\"a.ts\\"}"');
const deltaBIndex = body.indexOf('"partial_json":"{\\"path\\":\\"b.ts\\"}"');
expect(deltaAIndex).toBeGreaterThan(-1);
expect(deltaBIndex).toBeGreaterThan(-1);
expect(stopIndex).toBeGreaterThan(deltaBIndex);
});
});
@@ -120,8 +120,48 @@ describe('ProxyRequestTransformer regressions', () => {
},
],
})
).toThrow('text is not allowed before tool_result blocks for pending tool_use ids');
expect(() =>
new ProxyRequestTransformer().transform({
messages: [
{
role: 'assistant',
content: [{ type: 'tool_use', id: 'toolu_1', name: 'vision', input: { detail: 'high' } }],
},
{
role: 'user',
content: [
{ type: 'tool_result', tool_use_id: 'toolu_1', content: 'result' },
{ type: 'text', text: 'follow-up' },
],
},
],
})
).not.toThrow();
expect(() =>
new ProxyRequestTransformer().transform({
messages: [
{
role: 'assistant',
content: [
{ type: 'tool_use', id: 'toolu_1', name: 'search', input: { q: 'docs' } },
{ type: 'tool_use', id: 'toolu_2', name: 'open', input: { url: 'https://example.com' } },
],
},
{
role: 'user',
content: [
{ type: 'tool_result', tool_use_id: 'toolu_1', content: 'partial' },
{ type: 'text', text: 'follow-up' },
{ type: 'tool_result', tool_use_id: 'toolu_2', content: 'done' },
],
},
],
})
).toThrow('text is not allowed between tool_result blocks for pending tool_use ids');
expect(() =>
new ProxyRequestTransformer().transform({
messages: [