From 301db6d76c4c0685c47325d8feea67d62cbda322 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Wed, 28 May 2025 07:16:19 -0700 Subject: [PATCH] responses.resolved_model column and response.set_resolved_model(model_id) method, closes #1117 --- README.md | 3 +++ docs/logging.md | 3 ++- docs/plugins/advanced-model-plugins.md | 12 ++++++++++++ llm/cli.py | 12 +++++++++++- llm/migrations.py | 7 +++++++ llm/models.py | 5 +++++ tests/conftest.py | 3 +++ tests/test_chat.py | 6 ++++++ tests/test_llm_logs.py | 24 ++++++++++++++++++++++++ tests/test_migrate.py | 1 + 10 files changed, 74 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 35da50e..a58782f 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,7 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. * [Plugin directory](https://llm.datasette.io/en/stable/plugins/directory.html) * [Local models](https://llm.datasette.io/en/stable/plugins/directory.html#local-models) * [Remote APIs](https://llm.datasette.io/en/stable/plugins/directory.html#remote-apis) + * [Tools](https://llm.datasette.io/en/stable/plugins/directory.html#tools) * [Fragments and template loaders](https://llm.datasette.io/en/stable/plugins/directory.html#fragments-and-template-loaders) * [Embedding models](https://llm.datasette.io/en/stable/plugins/directory.html#embedding-models) * [Extra commands](https://llm.datasette.io/en/stable/plugins/directory.html#extra-commands) @@ -253,6 +254,7 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. * [Plugin hooks](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html) * [register_commands(cli)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-commands-cli) * [register_models(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-models-register) + * [register_embedding_models(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-embedding-models-register) * [register_tools(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-tools-register) * [register_template_loaders(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-template-loaders-register) * [register_fragment_loaders(register)](https://llm.datasette.io/en/stable/plugins/plugin-hooks.html#register-fragment-loaders-register) @@ -278,6 +280,7 @@ See also [the llm tag](https://simonwillison.net/tags/llm/) on my blog. * [Supporting tools](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#supporting-tools) * [Attachments for multi-modal models](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#attachments-for-multi-modal-models) * [Tracking token usage](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-token-usage) + * [Tracking resolved model names](https://llm.datasette.io/en/stable/plugins/advanced-model-plugins.html#tracking-resolved-model-names) * [Utility functions for plugins](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html) * [llm.get_key()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-get-key) * [llm.user_dir()](https://llm.datasette.io/en/stable/plugins/plugin-utilities.html#llm-user-dir) diff --git a/docs/logging.md b/docs/logging.md index b93d870..4ef6746 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -327,7 +327,8 @@ CREATE TABLE "responses" ( [input_tokens] INTEGER, [output_tokens] INTEGER, [token_details] TEXT, - [schema_id] TEXT REFERENCES [schemas]([id]) + [schema_id] TEXT REFERENCES [schemas]([id]), + [resolved_model] TEXT ); CREATE VIRTUAL TABLE [responses_fts] USING FTS5 ( [prompt], diff --git a/docs/plugins/advanced-model-plugins.md b/docs/plugins/advanced-model-plugins.md index 13c4bc0..6a681b0 100644 --- a/docs/plugins/advanced-model-plugins.md +++ b/docs/plugins/advanced-model-plugins.md @@ -279,3 +279,15 @@ This example logs 15 input tokens, 340 output tokens and notes that 37 tokens we ```python response.set_usage(input=15, output=340, details={"cached": 37}) ``` +(advanced-model-plugins-resolved-model)= + +## Tracking resolved model names + +In some cases the model ID that the user requested may not be the exact model that is executed. Many providers have a `model-latest` alias which may execute different models over time. + +If those APIs return the _real_ model ID that was used, your plugin can record that in the `resources.resolved_model` column in the logs by calling this method and passing the string representing the resolved, final model ID: + +```bash +response.set_resolved_model(resolved_model_id) +``` +This string will be recorded in the database and shown in the output of `llm logs` and `llm logs --json`. \ No newline at end of file diff --git a/llm/cli.py b/llm/cli.py index 5d53e74..ad8379f 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -1390,6 +1390,7 @@ def logs_turn_off(): LOGS_COLUMNS = """ responses.id, responses.model, + responses.resolved_model, responses.prompt, responses.system, responses.prompt_json, @@ -2011,7 +2012,16 @@ def logs_list( else "" ), ( - "\nModel: **{}**\n".format(row["model"]) + ( + "\nModel: **{}**{}\n".format( + row["model"], + ( + " (resolved: **{}**)".format(row["resolved_model"]) + if row["resolved_model"] + else "" + ), + ) + ) if should_show_conversation else "" ), diff --git a/llm/migrations.py b/llm/migrations.py index b4dc124..ccbd806 100644 --- a/llm/migrations.py +++ b/llm/migrations.py @@ -390,3 +390,10 @@ def m018_tool_instances(db): ) # We record which instance was used only on the results db["tool_results"].add_column("instance_id", fk="tool_instances") + + +@migration +def m019_resolved_model(db): + # For models like gemini-1.5-flash-latest where we wish to record + # the resolved model name in addition to the alias + db["responses"].add_column("resolved_model", str) diff --git a/llm/models.py b/llm/models.py index 7329ee5..d31faca 100644 --- a/llm/models.py +++ b/llm/models.py @@ -568,6 +568,7 @@ class _BaseResponse: id: str prompt: "Prompt" stream: bool + resolved_model: Optional[str] = None conversation: Optional["_BaseConversation"] = None _key: Optional[str] = None _tool_calls: List[ToolCall] = [] @@ -620,6 +621,9 @@ class _BaseResponse: self.output_tokens = output self.token_details = details + def set_resolved_model(self, model_id: str): + self.resolved_model = model_id + @classmethod def from_row(cls, db, row, _async=False): from llm import get_model, get_async_model @@ -814,6 +818,7 @@ class _BaseResponse: json.dumps(self.token_details) if self.token_details else None ), "schema_id": schema_id, + "resolved_model": self.resolved_model, } db["responses"].insert(response) diff --git a/tests/conftest.py b/tests/conftest.py index d19f1f2..64ed5b6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -61,6 +61,7 @@ class MockModel(llm.Model): def __init__(self): self.history = [] self._queue = [] + self.resolved_model_name = None def enqueue(self, messages): assert isinstance(messages, list) @@ -81,6 +82,8 @@ class MockModel(llm.Model): response.set_usage( input=len((prompt.prompt or "").split()), output=len(gathered) ) + if self.resolved_model_name is not None: + response.set_resolved_model(self.resolved_model_name) class MockKeyModel(llm.KeyModel): diff --git a/tests/test_chat.py b/tests/test_chat.py index 382dd21..a0d010c 100644 --- a/tests/test_chat.py +++ b/tests/test_chat.py @@ -46,6 +46,7 @@ def test_chat_basic(mock_model, logs_db): { "id": ANY, "model": "mock", + "resolved_model": None, "prompt": "Hi", "system": None, "prompt_json": None, @@ -63,6 +64,7 @@ def test_chat_basic(mock_model, logs_db): { "id": ANY, "model": "mock", + "resolved_model": None, "prompt": "Hi two", "system": None, "prompt_json": None, @@ -110,6 +112,7 @@ def test_chat_basic(mock_model, logs_db): { "id": ANY, "model": "mock", + "resolved_model": None, "prompt": "Continue", "system": None, "prompt_json": None, @@ -153,6 +156,7 @@ def test_chat_system(mock_model, logs_db): { "id": ANY, "model": "mock", + "resolved_model": None, "prompt": "Hi", "system": "You are mean", "prompt_json": None, @@ -195,6 +199,7 @@ def test_chat_options(mock_model, logs_db, user_path): { "id": ANY, "model": "mock", + "resolved_model": None, "prompt": "Hi", "system": None, "prompt_json": None, @@ -212,6 +217,7 @@ def test_chat_options(mock_model, logs_db, user_path): { "id": ANY, "model": "mock", + "resolved_model": None, "prompt": "Hi with override", "system": None, "prompt_json": None, diff --git a/tests/test_llm_logs.py b/tests/test_llm_logs.py index 6b69343..5c4a387 100644 --- a/tests/test_llm_logs.py +++ b/tests/test_llm_logs.py @@ -944,3 +944,27 @@ def test_logs_backup(logs_db): assert result.output.startswith("Backed up ") assert result.output.endswith("to backup.db\n") assert expected_path.exists() + + +def test_logs_resolved_model(logs_db, mock_model): + mock_model.resolved_model_name = "resolved-mock" + runner = CliRunner() + result = runner.invoke(cli, ["-m", "mock", "simple prompt"]) + assert result.exit_code == 0 + # Should have logged the resolved model name + assert logs_db["responses"].count + response = list(logs_db["responses"].rows)[0] + assert response["model"] == "mock" + assert response["resolved_model"] == "resolved-mock" + + # Should show up in the JSON logs + result2 = runner.invoke(cli, ["logs", "--json"]) + assert result2.exit_code == 0 + logs = json.loads(result2.output.strip()) + assert len(logs) == 1 + assert logs[0]["model"] == "mock" + assert logs[0]["resolved_model"] == "resolved-mock" + + # And the rendered logs + result3 = runner.invoke(cli, ["logs"]) + assert "Model: **mock** (resolved: **resolved-mock**)" in result3.output diff --git a/tests/test_migrate.py b/tests/test_migrate.py index 2559b85..9108b1e 100644 --- a/tests/test_migrate.py +++ b/tests/test_migrate.py @@ -8,6 +8,7 @@ import sqlite_utils EXPECTED = { "id": str, "model": str, + "resolved_model": str, "prompt": str, "system": str, "prompt_json": str,