233 lines
8.2 KiB
Python
233 lines
8.2 KiB
Python
"""Critical tests for tool registry - Tests that would have caught bugs."""
|
|
|
|
import inspect
|
|
|
|
import pytest
|
|
|
|
from agent.prompts import PromptBuilder
|
|
from agent.registry import Tool, _create_tool_from_function, make_tools
|
|
|
|
|
|
class TestToolSpecFormat:
|
|
"""Critical tests for tool specification format."""
|
|
|
|
def test_tool_spec_format_is_openai_compatible(self):
|
|
"""CRITICAL: Verify tool specs are OpenAI-compatible."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
specs = builder.build_tools_spec()
|
|
|
|
# Verify structure
|
|
assert isinstance(specs, list), "Tool specs must be a list"
|
|
assert len(specs) > 0, "Tool specs list is empty"
|
|
|
|
for spec in specs:
|
|
# OpenAI format requires these fields
|
|
assert (
|
|
spec["type"] == "function"
|
|
), f"Tool type must be 'function', got {spec.get('type')}"
|
|
assert "function" in spec, "Tool spec missing 'function' key"
|
|
|
|
func = spec["function"]
|
|
assert "name" in func, "Function missing 'name'"
|
|
assert "description" in func, "Function missing 'description'"
|
|
assert "parameters" in func, "Function missing 'parameters'"
|
|
|
|
params = func["parameters"]
|
|
assert params["type"] == "object", "Parameters type must be 'object'"
|
|
assert "properties" in params, "Parameters missing 'properties'"
|
|
assert "required" in params, "Parameters missing 'required'"
|
|
assert isinstance(params["required"], list), "Required must be a list"
|
|
|
|
def test_tool_parameters_match_function_signature(self):
|
|
"""CRITICAL: Verify generated parameters match function signature."""
|
|
|
|
def test_func(name: str, age: int, active: bool = True):
|
|
"""Test function with typed parameters."""
|
|
return {"status": "ok"}
|
|
|
|
tool = _create_tool_from_function(test_func)
|
|
|
|
# Verify types are correctly mapped
|
|
assert tool.parameters["properties"]["name"]["type"] == "string"
|
|
assert tool.parameters["properties"]["age"]["type"] == "integer"
|
|
assert tool.parameters["properties"]["active"]["type"] == "boolean"
|
|
|
|
# Verify required vs optional
|
|
assert "name" in tool.parameters["required"], "name should be required"
|
|
assert "age" in tool.parameters["required"], "age should be required"
|
|
assert (
|
|
"active" not in tool.parameters["required"]
|
|
), "active has default, should not be required"
|
|
|
|
def test_all_registered_tools_are_callable(self):
|
|
"""CRITICAL: Verify all registered tools are actually callable."""
|
|
tools = make_tools()
|
|
|
|
assert len(tools) > 0, "No tools registered"
|
|
|
|
for name, tool in tools.items():
|
|
assert callable(tool.func), f"Tool {name} is not callable"
|
|
|
|
# Verify function has valid signature
|
|
try:
|
|
inspect.signature(tool.func)
|
|
# If we get here, signature is valid
|
|
except Exception as e:
|
|
pytest.fail(f"Tool {name} has invalid signature: {e}")
|
|
|
|
def test_tools_spec_contains_all_registered_tools(self):
|
|
"""CRITICAL: Verify build_tools_spec() returns all registered tools."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
specs = builder.build_tools_spec()
|
|
|
|
spec_names = {spec["function"]["name"] for spec in specs}
|
|
tool_names = set(tools.keys())
|
|
|
|
missing = tool_names - spec_names
|
|
extra = spec_names - tool_names
|
|
|
|
assert not missing, f"Tools missing from specs: {missing}"
|
|
assert not extra, f"Extra tools in specs: {extra}"
|
|
assert spec_names == tool_names, "Tool specs don't match registered tools"
|
|
|
|
def test_tool_description_extracted_from_docstring(self):
|
|
"""Verify tool description is extracted from function docstring."""
|
|
|
|
def test_func(param: str):
|
|
"""This is the description.
|
|
|
|
More details here.
|
|
"""
|
|
return {}
|
|
|
|
tool = _create_tool_from_function(test_func)
|
|
|
|
assert tool.description == "This is the description."
|
|
assert "More details" not in tool.description
|
|
|
|
def test_tool_without_docstring_uses_function_name(self):
|
|
"""Verify tool without docstring uses function name as description."""
|
|
|
|
def test_func_no_doc(param: str):
|
|
return {}
|
|
|
|
tool = _create_tool_from_function(test_func_no_doc)
|
|
|
|
assert tool.description == "test_func_no_doc"
|
|
|
|
def test_tool_parameters_have_descriptions(self):
|
|
"""Verify all tool parameters have descriptions."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
specs = builder.build_tools_spec()
|
|
|
|
for spec in specs:
|
|
params = spec["function"]["parameters"]
|
|
properties = params.get("properties", {})
|
|
|
|
for param_name, param_spec in properties.items():
|
|
assert (
|
|
"description" in param_spec
|
|
), f"Parameter {param_name} in {spec['function']['name']} missing description"
|
|
|
|
def test_required_parameters_are_marked_correctly(self):
|
|
"""Verify required parameters are correctly identified."""
|
|
|
|
def func_with_optional(required: str, optional: int = 5):
|
|
return {}
|
|
|
|
tool = _create_tool_from_function(func_with_optional)
|
|
|
|
assert "required" in tool.parameters["required"]
|
|
assert "optional" not in tool.parameters["required"]
|
|
assert len(tool.parameters["required"]) == 1
|
|
|
|
|
|
class TestToolRegistry:
|
|
"""Tests for tool registry functionality."""
|
|
|
|
def test_make_tools_returns_dict(self):
|
|
"""Verify make_tools returns a dictionary."""
|
|
tools = make_tools()
|
|
|
|
assert isinstance(tools, dict)
|
|
assert len(tools) > 0
|
|
|
|
def test_all_tools_have_unique_names(self):
|
|
"""Verify all tool names are unique."""
|
|
tools = make_tools()
|
|
|
|
names = [tool.name for tool in tools.values()]
|
|
assert len(names) == len(set(names)), "Duplicate tool names found"
|
|
|
|
def test_tool_names_match_dict_keys(self):
|
|
"""Verify tool names match their dictionary keys."""
|
|
tools = make_tools()
|
|
|
|
for key, tool in tools.items():
|
|
assert key == tool.name, f"Key {key} doesn't match tool name {tool.name}"
|
|
|
|
def test_expected_tools_are_registered(self):
|
|
"""Verify all expected tools are registered."""
|
|
tools = make_tools()
|
|
|
|
expected_tools = [
|
|
"set_path_for_folder",
|
|
"list_folder",
|
|
"find_media_imdb_id",
|
|
"find_torrent",
|
|
"add_torrent_by_index",
|
|
"add_torrent_to_qbittorrent",
|
|
"get_torrent_by_index",
|
|
"set_language",
|
|
]
|
|
|
|
for expected in expected_tools:
|
|
assert expected in tools, f"Expected tool {expected} not registered"
|
|
|
|
def test_tool_functions_are_valid(self):
|
|
"""Verify all tool functions are properly structured."""
|
|
tools = make_tools()
|
|
|
|
# Verify structure without calling functions
|
|
# (calling would require full setup with memory, clients, etc.)
|
|
for name, tool in tools.items():
|
|
assert callable(tool.func), f"Tool {name} function is not callable"
|
|
|
|
|
|
class TestToolDataclass:
|
|
"""Tests for Tool dataclass."""
|
|
|
|
def test_tool_creation(self):
|
|
"""Verify Tool can be created with all fields."""
|
|
|
|
def dummy_func():
|
|
return {}
|
|
|
|
tool = Tool(
|
|
name="test_tool",
|
|
description="Test description",
|
|
func=dummy_func,
|
|
parameters={"type": "object", "properties": {}, "required": []},
|
|
)
|
|
|
|
assert tool.name == "test_tool"
|
|
assert tool.description == "Test description"
|
|
assert tool.func == dummy_func
|
|
assert isinstance(tool.parameters, dict)
|
|
|
|
def test_tool_parameters_structure(self):
|
|
"""Verify Tool parameters have correct structure."""
|
|
|
|
def dummy_func(arg: str):
|
|
return {}
|
|
|
|
tool = _create_tool_from_function(dummy_func)
|
|
|
|
assert "type" in tool.parameters
|
|
assert "properties" in tool.parameters
|
|
assert "required" in tool.parameters
|
|
assert tool.parameters["type"] == "object"
|