mirror of
https://github.com/Hopiu/llm.git
synced 2026-04-27 00:14:46 +00:00
2f206d0e26
Also includes a bug fix for system fragments, see https://github.com/simonw/llm/issues/1240#issuecomment-3177684518
62 lines
2 KiB
Python
62 lines
2 KiB
Python
from click.testing import CliRunner
|
|
import sys
|
|
import llm.cli
|
|
import pytest
|
|
|
|
|
|
@pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows")
|
|
def test_chat_template_system_only_no_duplicate_prompt(
|
|
mock_model, logs_db, templates_path
|
|
):
|
|
# Template that only sets a system prompt, no user prompt
|
|
(templates_path / "wild-french.yaml").write_text(
|
|
"system: Speak in French\n", "utf-8"
|
|
)
|
|
|
|
runner = CliRunner()
|
|
mock_model.enqueue(["Bonjour !"])
|
|
result = runner.invoke(
|
|
llm.cli.cli,
|
|
["chat", "-m", "mock", "-t", "wild-french"],
|
|
input="hi\nquit\n",
|
|
catch_exceptions=False,
|
|
)
|
|
assert result.exit_code == 0
|
|
|
|
# Ensure the logged prompt is not duplicated (no "hi\nhi")
|
|
rows = list(logs_db["responses"].rows)
|
|
assert len(rows) == 1
|
|
assert rows[0]["prompt"] == "hi"
|
|
assert rows[0]["system"] == "Speak in French"
|
|
|
|
|
|
@pytest.mark.xfail(sys.platform == "win32", reason="Expected to fail on Windows")
|
|
def test_chat_system_fragments_only_first_turn(tmpdir, mock_model, logs_db):
|
|
# Create a system fragment file
|
|
sys_frag_path = str(tmpdir / "sys.txt")
|
|
with open(sys_frag_path, "w", encoding="utf-8") as fp:
|
|
fp.write("System fragment content")
|
|
|
|
runner = CliRunner()
|
|
# Two responses queued for two turns
|
|
mock_model.enqueue(["first"])
|
|
mock_model.enqueue(["second"])
|
|
result = runner.invoke(
|
|
llm.cli.cli,
|
|
["chat", "-m", "mock", "--system-fragment", sys_frag_path],
|
|
input="Hi\nHi two\nquit\n",
|
|
catch_exceptions=False,
|
|
)
|
|
assert result.exit_code == 0
|
|
|
|
# Verify only the first response has the system fragment
|
|
responses = list(logs_db["responses"].rows)
|
|
assert len(responses) == 2
|
|
first_id = responses[0]["id"]
|
|
second_id = responses[1]["id"]
|
|
|
|
sys_frags = list(logs_db["system_fragments"].rows)
|
|
# Exactly one system fragment row, attached to the first response only
|
|
assert len(sys_frags) == 1
|
|
assert sys_frags[0]["response_id"] == first_id
|
|
assert sys_frags[0]["response_id"] != second_id
|