llm tools and llm tools --json, closes #994

llm/cli.py

@@ -60,6 +60,7 @@ from .utils import (
 )
 import base64
 import httpx
+import inspect
 import pathlib
 import pydantic
 import re
@@ -2097,6 +2098,43 @@ def schemas_dsl_debug(input, multi):
     click.echo(json.dumps(schema, indent=2))
 
 
+@cli.group(
+    cls=DefaultGroup,
+    default="list",
+    default_if_no_args=True,
+)
+def tools():
+    "Manage tools that can be made available to LLMs"
+
+
+@tools.command(name="list")
+@click.option("json_", "--json", is_flag=True, help="Output as JSON")
+def tools_list(json_):
+    "List available tools that have been provided by plugins"
+    tools = get_tools()
+    if json_:
+        click.echo(
+            json.dumps(
+                {
+                    name: {
+                        "description": tool.description,
+                        "arguments": tool.input_schema,
+                    }
+                    for name, tool in tools.items()
+                },
+                indent=2,
+            )
+        )
+    else:
+        for name, tool in tools.items():
+            sig = "()"
+            if tool.implementation:
+                sig = str(inspect.signature(tool.implementation))
+            click.echo("{}{}".format(name, sig))
+            if tool.description:
+                click.echo(textwrap.indent(tool.description, "  "))
+
+
 @cli.group(
     cls=DefaultGroup,
     default="list",

tests/test_plugins.py

@@ -1,6 +1,7 @@
 from click.testing import CliRunner
 import click
 import importlib
+import json
 import llm
 from llm import cli, hookimpl, plugins, get_template_loaders, get_fragment_loaders
 import textwrap
@@ -235,6 +236,40 @@ def test_register_tools():
                 implementation=count_character_in_word,
             ),
         }
+        # Test the CLI command
+        runner = CliRunner()
+        result = runner.invoke(cli.cli, ["tools", "list"])
+        assert result.exit_code == 0
+        assert result.output == (
+            "upper(text: str) -> str\n"
+            "  Convert text to uppercase.\n"
+            "count_character_in_word(text: str, character: str) -> int\n"
+            "  Count the number of occurrences of a character in a word.\n"
+        )
+        # And --json
+        result = runner.invoke(cli.cli, ["tools", "list", "--json"])
+        assert result.exit_code == 0
+        assert json.loads(result.output) == {
+            "upper": {
+                "description": "Convert text to uppercase.",
+                "arguments": {
+                    "properties": {"text": {"type": "string"}},
+                    "required": ["text"],
+                    "type": "object",
+                },
+            },
+            "count_character_in_word": {
+                "description": "Count the number of occurrences of a character in a word.",
+                "arguments": {
+                    "properties": {
+                        "text": {"type": "string"},
+                        "character": {"type": "string"},
+                    },
+                    "required": ["text", "character"],
+                    "type": "object",
+                },
+            },
+        }
     finally:
         plugins.pm.unregister(name="ToolsPlugin")
         assert llm.get_tools() == {}