Hopiu/llm
1
0
Fork 0
mirror of https://github.com/Hopiu/llm.git synced 2026-04-10 00:10:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

responses.resolved_model column and response.set_resolved_model(model_id) method, closes #1117

Browse source
This commit is contained in:
Simon Willison 2025-05-28 07:16:19 -07:00
parent afb170a62a
commit 301db6d76c
10 changed files with 74 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
README.md
View file

@ -246,6 +246,7 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog.
* [Plugin directory](https://llm.datasette.io/en/stable/plugins/directory.html)
* [Local models](https://llm.datasette.io/en/stable/plugins/directory.html#local-models)
* [Remote APIs](https://llm.datasette.io/en/stable/plugins/directory.html#remote-apis)
* [Tools](https://llm.datasette.io/en/stable/plugins/directory.html#tools)
* [Fragments and template loaders](https://llm.datasette.io/en/stable/plugins/directory.html#fragments-and-template-loaders)
* [Embedding models](https://llm.datasette.io/en/stable/plugins/directory.html#embedding-models)
* [Extra commands](https://llm.datasette.io/en/stable/plugins/directory.html#extra-commands)
@ -253,6 +254,7 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog.
* [Plugin hooks](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html)
* [register_commands(cli)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-commands-cli)
* [register_models(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-models-register)
* [register_embedding_models(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-embedding-models-register)
* [register_tools(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-tools-register)
* [register_template_loaders(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-template-loaders-register)
* [register_fragment_loaders(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-fragment-loaders-register)
@ -278,6 +280,7 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog.
* [Supporting tools](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-tools)
* [Attachments for multi-modal models](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#attachments-for-multi-modal-models)
* [Tracking token usage](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-token-usage)
* [Tracking resolved model names](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-resolved-model-names)
* [Utility functions for plugins](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html)
* [llm.get_key()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-get-key)
* [llm.user_dir()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-user-dir)

3
docs/logging.md
View file

@ -327,7 +327,8 @@ CREATE TABLE "responses" (
[input_tokens] INTEGER,
[output_tokens] INTEGER,
[token_details] TEXT,
[schema_id] TEXT REFERENCES [schemas]([id])
[schema_id] TEXT REFERENCES [schemas]([id]),
[resolved_model] TEXT
);
CREATE VIRTUAL TABLE [responses_fts] USING FTS5 (
[prompt],

12
docs/plugins/advanced-model-plugins.md
View file

@ -279,3 +279,15 @@ This example logs 15 input tokens, 340 output tokens and notes that 37 tokens we
```python
response.set_usage(input=15, output=340, details={"cached": 37})
```
(advanced-model-plugins-resolved-model)=
## Tracking resolved model names
In some cases the model ID that the user requested may not be the exact model that is executed. Many providers have a `model-latest` alias which may execute different models over time.
If those APIs return the _real_ model ID that was used, your plugin can record that in the `resources.resolved_model` column in the logs by calling this method and passing the string representing the resolved, final model ID:
```bash
response.set_resolved_model(resolved_model_id)
```
This string will be recorded in the database and shown in the output of `llm logs` and `llm logs --json`.

12
llm/cli.py
View file

@ -1390,6 +1390,7 @@ def logs_turn_off():
LOGS_COLUMNS = """ responses.id,
responses.model,
responses.resolved_model,
responses.prompt,
responses.system,
responses.prompt_json,
@ -2011,7 +2012,16 @@ def logs_list(
else ""
),
(
"\nModel: **{}**\n".format(row["model"])
(
"\nModel: **{}**{}\n".format(
row["model"],
(
" (resolved: **{}**)".format(row["resolved_model"])
if row["resolved_model"]
else ""
),
)
)
if should_show_conversation
else ""
),

7
llm/migrations.py
View file

@ -390,3 +390,10 @@ def m018_tool_instances(db):
)
# We record which instance was used only on the results
db["tool_results"].add_column("instance_id", fk="tool_instances")
@migration
def m019_resolved_model(db):
# For models like gemini-1.5-flash-latest where we wish to record
# the resolved model name in addition to the alias
db["responses"].add_column("resolved_model", str)

5
llm/models.py
View file

@ -568,6 +568,7 @@ class _BaseResponse:
id: str
prompt: "Prompt"
stream: bool
resolved_model: Optional[str] = None
conversation: Optional["_BaseConversation"] = None
_key: Optional[str] = None
_tool_calls: List[ToolCall] = []
@ -620,6 +621,9 @@ class _BaseResponse:
self.output_tokens = output
self.token_details = details
def set_resolved_model(self, model_id: str):
self.resolved_model = model_id
@classmethod
def from_row(cls, db, row, _async=False):
from llm import get_model, get_async_model
@ -814,6 +818,7 @@ class _BaseResponse:
json.dumps(self.token_details) if self.token_details else None
),
"schema_id": schema_id,
"resolved_model": self.resolved_model,
}
db["responses"].insert(response)

3
tests/conftest.py
View file

@ -61,6 +61,7 @@ class MockModel(llm.Model):
def __init__(self):
self.history = []
self._queue = []
self.resolved_model_name = None
def enqueue(self, messages):
assert isinstance(messages, list)
@ -81,6 +82,8 @@ class MockModel(llm.Model):
response.set_usage(
input=len((prompt.prompt or "").split()), output=len(gathered)
)
if self.resolved_model_name is not None:
response.set_resolved_model(self.resolved_model_name)
class MockKeyModel(llm.KeyModel):

6
tests/test_chat.py
View file

@ -46,6 +46,7 @@ def test_chat_basic(mock_model, logs_db):
{
"id": ANY,
"model": "mock",
"resolved_model": None,
"prompt": "Hi",
"system": None,
"prompt_json": None,
@ -63,6 +64,7 @@ def test_chat_basic(mock_model, logs_db):
{
"id": ANY,
"model": "mock",
"resolved_model": None,
"prompt": "Hi two",
"system": None,
"prompt_json": None,
@ -110,6 +112,7 @@ def test_chat_basic(mock_model, logs_db):
{
"id": ANY,
"model": "mock",
"resolved_model": None,
"prompt": "Continue",
"system": None,
"prompt_json": None,
@ -153,6 +156,7 @@ def test_chat_system(mock_model, logs_db):
{
"id": ANY,
"model": "mock",
"resolved_model": None,
"prompt": "Hi",
"system": "You are mean",
"prompt_json": None,
@ -195,6 +199,7 @@ def test_chat_options(mock_model, logs_db, user_path):
{
"id": ANY,
"model": "mock",
"resolved_model": None,
"prompt": "Hi",
"system": None,
"prompt_json": None,
@ -212,6 +217,7 @@ def test_chat_options(mock_model, logs_db, user_path):
{
"id": ANY,
"model": "mock",
"resolved_model": None,
"prompt": "Hi with override",
"system": None,
"prompt_json": None,

24
tests/test_llm_logs.py
View file

@ -944,3 +944,27 @@ def test_logs_backup(logs_db):
assert result.output.startswith("Backed up ")
assert result.output.endswith("to backup.db\n")
assert expected_path.exists()
def test_logs_resolved_model(logs_db, mock_model):
mock_model.resolved_model_name = "resolved-mock"
runner = CliRunner()
result = runner.invoke(cli, ["-m", "mock", "simple prompt"])
assert result.exit_code == 0
# Should have logged the resolved model name
assert logs_db["responses"].count
response = list(logs_db["responses"].rows)[0]
assert response["model"] == "mock"
assert response["resolved_model"] == "resolved-mock"
# Should show up in the JSON logs
result2 = runner.invoke(cli, ["logs", "--json"])
assert result2.exit_code == 0
logs = json.loads(result2.output.strip())
assert len(logs) == 1
assert logs[0]["model"] == "mock"
assert logs[0]["resolved_model"] == "resolved-mock"
# And the rendered logs
result3 = runner.invoke(cli, ["logs"])
assert "Model: **mock** (resolved: **resolved-mock**)" in result3.output

1
tests/test_migrate.py
View file

@ -8,6 +8,7 @@ import sqlite_utils
EXPECTED = {
"id": str,
"model": str,
"resolved_model": str,
"prompt": str,
"system": str,
"prompt_json": str,