From 2a54939951ceed7137066f337e6382024eb99e68 Mon Sep 17 00:00:00 2001 From: James Sanford Date: Mon, 11 Aug 2025 14:17:52 -0700 Subject: [PATCH] Fix streaming tool calls with tests for many variants. (#1218) * Recorded instance of streaming tool response variant "a". This is the typical response, where "arguments":"" arrives first in the stream, followed by "arguments":"{}" The response data is a real capture from the OpenRouter API, however some request and header data may be from other test fixtures. * Recorded instance of streaming tool response variant "b". This is a streaming response where the first arguments you get is a fully formed "arguments":"{}" The response data is a real capture from the OpenRouter API, however some request and header data may be from other test fixtures. * Test cases for streaming tool responses. Note that the replays are marked as "read-only", as they are variants seen in the wild where the streaming tool call argument fragments arrive in a specific order. * Fix streaming tool response variant "b", where "arguments":"{}" is what arrives first. The previous code erroneously caused the first "arguments" to be duplicated, by using "+=" even when being initially set. This went unnoticed as many models stream "arguments":"" first. When a more fully formed "arguments" fragment arrived first, it was causing "Error: Extra data: line 1 column 3 (char 2)" * Recorded instance of streaming tool response variant "c". This was failing with "Error: unsupported operand type(s) for +=: 'NoneType' and 'str'" The response data is a real capture from the OpenRouter API, however some request and header data may be from other test fixtures. * Test case for streaming tool response variant "c". * Fix streaming tool response variant "c". However, I'm not sure why arguments was initially not present or seen as None. --- llm/default_plugins/openai_models.py | 18 +- .../test_tools_streaming_variant_a.yaml | 154 ++++++++++++++++++ .../test_tools_streaming_variant_b.yaml | 151 +++++++++++++++++ .../test_tools_streaming_variant_c.yaml | 149 +++++++++++++++++ tests/test_tools_streaming.py | 44 +++++ 5 files changed, 510 insertions(+), 6 deletions(-) create mode 100644 tests/cassettes/test_tools_streaming/test_tools_streaming_variant_a.yaml create mode 100644 tests/cassettes/test_tools_streaming/test_tools_streaming_variant_b.yaml create mode 100644 tests/cassettes/test_tools_streaming/test_tools_streaming_variant_c.yaml create mode 100644 tests/test_tools_streaming.py diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 0deb61b..94c1ffc 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -716,12 +716,15 @@ class Chat(_Shared, KeyModel): usage = chunk.usage.model_dump() if chunk.choices and chunk.choices[0].delta: for tool_call in chunk.choices[0].delta.tool_calls or []: + if tool_call.function.arguments is None: + tool_call.function.arguments = "" index = tool_call.index if index not in tool_calls: tool_calls[index] = tool_call - tool_calls[ - index - ].function.arguments += tool_call.function.arguments + else: + tool_calls[ + index + ].function.arguments += tool_call.function.arguments try: content = chunk.choices[0].delta.content except IndexError: @@ -800,12 +803,15 @@ class AsyncChat(_Shared, AsyncKeyModel): usage = chunk.usage.model_dump() if chunk.choices and chunk.choices[0].delta: for tool_call in chunk.choices[0].delta.tool_calls or []: + if tool_call.function.arguments is None: + tool_call.function.arguments = "" index = tool_call.index if index not in tool_calls: tool_calls[index] = tool_call - tool_calls[ - index - ].function.arguments += tool_call.function.arguments + else: + tool_calls[ + index + ].function.arguments += tool_call.function.arguments try: content = chunk.choices[0].delta.content except IndexError: diff --git a/tests/cassettes/test_tools_streaming/test_tools_streaming_variant_a.yaml b/tests/cassettes/test_tools_streaming/test_tools_streaming_variant_a.yaml new file mode 100644 index 0000000..41bf259 --- /dev/null +++ b/tests/cassettes/test_tools_streaming/test_tools_streaming_variant_a.yaml @@ -0,0 +1,154 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":"What is the current llm version?"}],"model":"gpt-4.1-mini","stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"llm_version","description":"Return + the installed version of llm","parameters":{"properties":{},"type":"object"}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '315' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.78.0 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"gen-1753242299-QZRAt5HJHd1ptY8sdS0s","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242299-QZRAt5HJHd1ptY8sdS0s","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242299-QZRAt5HJHd1ptY8sdS0s","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"0","type":"function","function":{"name":"llm_version","arguments":""}}]},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242299-QZRAt5HJHd1ptY8sdS0s","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"0","type":"function","function":{"name":"llm_version","arguments":"{}"}}]},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242299-QZRAt5HJHd1ptY8sdS0s","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"usage":{"prompt_tokens":57,"completion_tokens":17,"total_tokens":74,"cost":0.00007159,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0},"cost_details":{"upstream_inference_cost":null},"completion_tokens_details":{"reasoning_tokens":0}}} + + + data: [DONE] + + + ' + headers: + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Tue, 23 Jul 2025 14:54:09 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"role":"user","content":"What is the current llm version?"},{"role":"assistant","content":""},{"role":"assistant","tool_calls":[{"type":"function","id":"0","function":{"name":"llm_version","arguments":"{}"}}]},{"role":"tool","tool_call_id":"0","content":"0.26"}],"model":"gpt-4.1-mini","stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"llm_version","description":"Return + the installed version of llm","parameters":{"properties":{},"type":"object"}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '517' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.78.0 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"The"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + current"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + version"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + of"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + *"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"ll"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"m"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"*"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + is"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + **"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"0"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"."},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"26"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"**."},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"stop","native_finish_reason":"stop","logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"usage":{"prompt_tokens":107,"completion_tokens":15,"total_tokens":122,"cost":0.0001017,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0},"cost_details":{"upstream_inference_cost":null},"completion_tokens_details":{"reasoning_tokens":0}}} + + + data: [DONE] + + + ' + headers: + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Tue, 23 Jul 2025 14:54:10 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/test_tools_streaming/test_tools_streaming_variant_b.yaml b/tests/cassettes/test_tools_streaming/test_tools_streaming_variant_b.yaml new file mode 100644 index 0000000..062c9e6 --- /dev/null +++ b/tests/cassettes/test_tools_streaming/test_tools_streaming_variant_b.yaml @@ -0,0 +1,151 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":"What is the current llm version?"}],"model":"gpt-4.1-mini","stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"llm_version","description":"Return + the installed version of llm","parameters":{"properties":{},"type":"object"}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '315' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.78.0 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"gen-1753242299-QZRAt5HJHd1ptY8sdS0s","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242299-QZRAt5HJHd1ptY8sdS0s","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242299-QZRAt5HJHd1ptY8sdS0s","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"0","type":"function","function":{"name":"llm_version","arguments":"{}"}}]},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":""} + + + data: {"id":"gen-1753242299-QZRAt5HJHd1ptY8sdS0s","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242299,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"usage":{"prompt_tokens":57,"completion_tokens":17,"total_tokens":74,"cost":0.00007159,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0},"cost_details":{"upstream_inference_cost":null},"completion_tokens_details":{"reasoning_tokens":0}}} + + + data: [DONE] + + + ' + headers: + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Tue, 23 Jul 2025 14:54:09 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"role":"user","content":"What is the current llm version?"},{"role":"assistant","content":""},{"role":"assistant","tool_calls":[{"type":"function","id":"0","function":{"name":"llm_version","arguments":"{}"}}]},{"role":"tool","tool_call_id":"0","content":"0.26"}],"model":"gpt-4.1-mini","stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"llm_version","description":"Return + the installed version of llm","parameters":{"properties":{},"type":"object"}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '517' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.78.0 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"The"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + current"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + version"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + of"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + *"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"ll"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"m"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"*"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + is"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":" + **"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"0"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"."},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"26"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":"**."},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"stop","native_finish_reason":"stop","logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753242300-j60LWi6MpN4lMZw1zTHK","provider":"Moonshot AI","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753242300,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"usage":{"prompt_tokens":107,"completion_tokens":15,"total_tokens":122,"cost":0.0001017,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0},"cost_details":{"upstream_inference_cost":null},"completion_tokens_details":{"reasoning_tokens":0}}} + + + data: [DONE] + + + ' + headers: + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Tue, 23 Jul 2025 14:54:10 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/test_tools_streaming/test_tools_streaming_variant_c.yaml b/tests/cassettes/test_tools_streaming/test_tools_streaming_variant_c.yaml new file mode 100644 index 0000000..31dc6b5 --- /dev/null +++ b/tests/cassettes/test_tools_streaming/test_tools_streaming_variant_c.yaml @@ -0,0 +1,149 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":"What is the current llm version?"}],"model":"gpt-4.1-mini","stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"llm_version","description":"Return + the installed version of llm","parameters":{"properties":{},"type":"object"}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '315' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.78.0 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: ' + data: {"id":"gen-1753248108-FGOxpkEzFEwhNKSPpI4a","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248108,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753248108-FGOxpkEzFEwhNKSPpI4a","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248108,"choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"llm_version:0","type":"function","function":{"name":"llm_version"}}]},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753248108-FGOxpkEzFEwhNKSPpI4a","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248108,"choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"function":{"arguments":"{}"},"type":"function"}]},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753248108-FGOxpkEzFEwhNKSPpI4a","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248108,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"tool_calls","native_finish_reason":"tool_calls","logprobs":null}],"system_fingerprint":"fpv0_170758dd"} + + + data: {"id":"gen-1753248108-FGOxpkEzFEwhNKSPpI4a","provider":"Novita","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248108,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"usage":{"prompt_tokens":56,"completion_tokens":12,"total_tokens":68,"cost":0.00005952,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0},"cost_details":{"upstream_inference_cost":null},"completion_tokens_details":{"reasoning_tokens":0}}} + + + data: [DONE] + + + ' + headers: + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Tue, 23 Jul 2025 14:54:09 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"role":"user","content":"What is the current llm version?"},{"role":"assistant","content":""},{"role":"assistant","tool_calls":[{"type":"function","id":"0","function":{"name":"llm_version","arguments":"{}"}}]},{"role":"tool","tool_call_id":"0","content":"0.26"}],"model":"gpt-4.1-mini","stream":true,"stream_options":{"include_usage":true},"tools":[{"type":"function","function":{"name":"llm_version","description":"Return + the installed version of llm","parameters":{"properties":{},"type":"object"}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '517' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.78.0 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]} + + + data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":"The installed"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]} + + + data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":" version"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]} + + + data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":" of"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]} + + + data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":" LL"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]} + + + data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":"M"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]} + + + data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":" on"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]} + + + data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":" this"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]} + + + data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":" system"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]} + + + data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":" is"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]} + + + data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":" "},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]} + + + data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":"0"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]} + + + data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":"."},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]} + + + data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":"26"},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]} + + + data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":"."},"finish_reason":null,"native_finish_reason":null,"logprobs":null}]} + + + data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":"stop","native_finish_reason":"stop","logprobs":null}]} + + + data: {"id":"gen-1753248104-uf1xqJDBrAUCJ4g8apK8","provider":"Fireworks","model":"moonshotai/kimi-k2","object":"chat.completion.chunk","created":1753248104,"choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null,"native_finish_reason":null,"logprobs":null}],"usage":{"prompt_tokens":105,"completion_tokens":16,"total_tokens":121,"cost":0.000103,"is_byok":false,"prompt_tokens_details":{"cached_tokens":0},"cost_details":{"upstream_inference_cost":null},"completion_tokens_details":{"reasoning_tokens":0}}} + + + data: [DONE] + + + ' + headers: + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Tue, 23 Jul 2025 14:54:10 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + status: + code: 200 + message: OK +version: 1 diff --git a/tests/test_tools_streaming.py b/tests/test_tools_streaming.py new file mode 100644 index 0000000..e9da7a1 --- /dev/null +++ b/tests/test_tools_streaming.py @@ -0,0 +1,44 @@ +from importlib.metadata import version +import llm +from llm.tools import llm_version +import os +import pytest + + +API_KEY = os.environ.get("PYTEST_OPENAI_API_KEY", None) or "badkey" + + +# This response contains streaming variant "a" where arguments="" is followed by arguments="{}" +@pytest.mark.vcr(record_mode="none") +def test_tools_streaming_variant_a(): + model = llm.get_model("gpt-4.1-mini") + chain = model.chain( + "What is the current llm version?", tools=[llm_version], key=API_KEY + ) + assert "".join(chain) == "The current version of *llm* is **{}**.".format( + version("llm") + ) + + +# This response contains streaming variant "b" where arguments="{}" is the first partial stream received. +@pytest.mark.vcr(record_mode="none") +def test_tools_streaming_variant_b(): + model = llm.get_model("gpt-4.1-mini") + chain = model.chain( + "What is the current llm version?", tools=[llm_version], key=API_KEY + ) + assert "".join(chain) == "The current version of *llm* is **{}**.".format( + version("llm") + ) + + +# This response contains streaming variant "c". +@pytest.mark.vcr(record_mode="none") +def test_tools_streaming_variant_c(): + model = llm.get_model("gpt-4.1-mini") + chain = model.chain( + "What is the current llm version?", tools=[llm_version], key=API_KEY + ) + assert "".join( + chain + ) == "The installed version of LLM on this system is {}.".format(version("llm"))