282 lines
9.3 KiB
Python
282 lines
9.3 KiB
Python
"""Critical tests for prompt builder - Tests that would have caught bugs."""
|
|
|
|
from alfred.agent.prompts import PromptBuilder
|
|
from alfred.agent.registry import make_tools
|
|
|
|
|
|
class TestPromptBuilderToolsInjection:
|
|
"""Critical tests for tools injection in prompts."""
|
|
|
|
def test_system_prompt_includes_all_tools(self, memory):
|
|
"""CRITICAL: Verify all tools are mentioned in system prompt."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
prompt = builder.build_system_prompt()
|
|
|
|
# Verify each tool is mentioned
|
|
for tool_name in tools.keys():
|
|
assert tool_name in prompt, (
|
|
f"Tool {tool_name} not mentioned in system prompt"
|
|
)
|
|
|
|
def test_tools_spec_contains_all_registered_tools(self, memory):
|
|
"""CRITICAL: Verify build_tools_spec() returns all tools."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
specs = builder.build_tools_spec()
|
|
|
|
spec_names = {spec["function"]["name"] for spec in specs}
|
|
tool_names = set(tools.keys())
|
|
|
|
assert spec_names == tool_names, f"Missing tools: {tool_names - spec_names}"
|
|
|
|
def test_tools_spec_is_not_empty(self, memory):
|
|
"""CRITICAL: Verify tools spec is never empty."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
specs = builder.build_tools_spec()
|
|
|
|
assert len(specs) > 0, "Tools spec is empty!"
|
|
|
|
def test_tools_spec_format_matches_openai(self, memory):
|
|
"""CRITICAL: Verify tools spec format is OpenAI-compatible."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
specs = builder.build_tools_spec()
|
|
|
|
for spec in specs:
|
|
assert "type" in spec
|
|
assert spec["type"] == "function"
|
|
assert "function" in spec
|
|
assert "name" in spec["function"]
|
|
assert "description" in spec["function"]
|
|
assert "parameters" in spec["function"]
|
|
|
|
|
|
class TestPromptBuilderMemoryContext:
|
|
"""Tests for memory context injection in prompts."""
|
|
|
|
def test_prompt_includes_current_topic(self, memory):
|
|
"""Verify current topic is included in prompt."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
|
|
memory.stm.set_topic("test_topic")
|
|
prompt = builder.build_system_prompt()
|
|
|
|
assert "test_topic" in prompt
|
|
|
|
def test_prompt_includes_extracted_entities(self, memory):
|
|
"""Verify extracted entities are included in prompt."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
|
|
memory.stm.set_entity("test_key", "test_value")
|
|
prompt = builder.build_system_prompt()
|
|
|
|
assert "test_key" in prompt
|
|
|
|
def test_prompt_includes_search_results(self, memory_with_search_results):
|
|
"""Verify search results are included in prompt."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
|
|
prompt = builder.build_system_prompt()
|
|
|
|
assert "Inception" in prompt
|
|
assert "LAST SEARCH" in prompt
|
|
|
|
def test_prompt_includes_active_downloads(self, memory):
|
|
"""Verify active downloads are included in prompt."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
|
|
memory.episodic.add_active_download(
|
|
{"task_id": "123", "name": "Test Movie", "progress": 50}
|
|
)
|
|
|
|
prompt = builder.build_system_prompt()
|
|
|
|
assert "ACTIVE DOWNLOADS" in prompt
|
|
assert "Test Movie" in prompt
|
|
|
|
def test_prompt_includes_recent_errors(self, memory):
|
|
"""Verify recent errors are included in prompt."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
|
|
memory.episodic.add_error("test_action", "test error message")
|
|
|
|
prompt = builder.build_system_prompt()
|
|
|
|
assert "RECENT ERRORS" in prompt or "error" in prompt.lower()
|
|
|
|
def test_prompt_includes_configuration(self, memory):
|
|
"""Verify configuration is included in prompt."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
|
|
memory.ltm.set_config("download_folder", "/test/downloads")
|
|
|
|
prompt = builder.build_system_prompt()
|
|
|
|
assert "CONFIGURATION" in prompt or "download_folder" in prompt
|
|
|
|
def test_prompt_includes_language(self, memory):
|
|
"""Verify language is included in prompt."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
|
|
memory.stm.set_language("fr")
|
|
|
|
prompt = builder.build_system_prompt()
|
|
|
|
assert "fr" in prompt or "LANGUAGE" in prompt
|
|
|
|
|
|
class TestPromptBuilderStructure:
|
|
"""Tests for prompt structure and completeness."""
|
|
|
|
def test_system_prompt_is_not_empty(self, memory):
|
|
"""Verify system prompt is never empty."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
prompt = builder.build_system_prompt()
|
|
|
|
assert len(prompt) > 0
|
|
assert prompt.strip() != ""
|
|
|
|
def test_system_prompt_includes_base_instruction(self, memory):
|
|
"""Verify system prompt includes base instruction."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
prompt = builder.build_system_prompt()
|
|
|
|
assert "assistant" in prompt.lower() or "help" in prompt.lower()
|
|
|
|
def test_system_prompt_includes_rules(self, memory):
|
|
"""Verify system prompt includes important rules."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
prompt = builder.build_system_prompt()
|
|
|
|
assert "RULES" in prompt or "IMPORTANT" in prompt
|
|
|
|
def test_system_prompt_includes_examples(self, memory):
|
|
"""Verify system prompt includes examples."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
prompt = builder.build_system_prompt()
|
|
|
|
assert "EXAMPLES" in prompt or "example" in prompt.lower()
|
|
|
|
def test_tools_description_format(self, memory):
|
|
"""Verify tools are properly formatted in description."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
|
|
description = builder._format_tools_description()
|
|
|
|
# Should have tool names and descriptions
|
|
for tool_name, _tool in tools.items():
|
|
assert tool_name in description
|
|
# Should have parameters info
|
|
assert "Parameters" in description or "parameters" in description
|
|
|
|
def test_episodic_context_format(self, memory_with_search_results):
|
|
"""Verify episodic context is properly formatted."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
|
|
context = builder._format_episodic_context(memory_with_search_results)
|
|
|
|
assert "LAST SEARCH" in context
|
|
assert "Inception" in context
|
|
|
|
def test_stm_context_format(self, memory):
|
|
"""Verify STM context is properly formatted."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
|
|
memory.stm.set_topic("test_topic")
|
|
memory.stm.set_entity("key", "value")
|
|
|
|
context = builder._format_stm_context(memory)
|
|
|
|
assert "TOPIC" in context or "test_topic" in context
|
|
assert "ENTITIES" in context or "key" in context
|
|
|
|
def test_config_context_format(self, memory):
|
|
"""Verify config context is properly formatted."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
|
|
memory.ltm.set_config("test_key", "test_value")
|
|
|
|
context = builder._format_config_context(memory)
|
|
|
|
assert "CONFIGURATION" in context
|
|
assert "test_key" in context
|
|
|
|
|
|
class TestPromptBuilderEdgeCases:
|
|
"""Tests for edge cases in prompt building."""
|
|
|
|
def test_prompt_with_no_memory_context(self, memory):
|
|
"""Verify prompt works with empty memory."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
|
|
# Memory is empty
|
|
prompt = builder.build_system_prompt()
|
|
|
|
# Should still have base content
|
|
assert len(prompt) > 0
|
|
assert "assistant" in prompt.lower()
|
|
|
|
def test_prompt_with_empty_tools(self):
|
|
"""Verify prompt handles empty tools dict."""
|
|
builder = PromptBuilder({})
|
|
|
|
prompt = builder.build_system_prompt()
|
|
|
|
# Should still generate a prompt
|
|
assert len(prompt) > 0
|
|
|
|
def test_tools_spec_with_empty_tools(self):
|
|
"""Verify tools spec handles empty tools dict."""
|
|
builder = PromptBuilder({})
|
|
|
|
specs = builder.build_tools_spec()
|
|
|
|
assert isinstance(specs, list)
|
|
assert len(specs) == 0
|
|
|
|
def test_prompt_with_unicode_in_memory(self, memory):
|
|
"""Verify prompt handles unicode in memory."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
|
|
memory.stm.set_entity("movie", "Amélie 🎬")
|
|
|
|
prompt = builder.build_system_prompt()
|
|
|
|
assert "Amélie" in prompt
|
|
assert "🎬" in prompt
|
|
|
|
def test_prompt_with_long_search_results(self, memory):
|
|
"""Verify prompt handles many search results."""
|
|
tools = make_tools()
|
|
builder = PromptBuilder(tools)
|
|
|
|
# Add many results
|
|
results = [{"name": f"Movie {i}", "seeders": i} for i in range(20)]
|
|
memory.episodic.store_search_results("test", results, "torrent")
|
|
|
|
prompt = builder.build_system_prompt()
|
|
|
|
# Should include some results but not all (to avoid huge prompts)
|
|
assert "Movie 0" in prompt or "Movie 1" in prompt
|
|
# Should indicate there are more
|
|
assert "more" in prompt.lower() or "..." in prompt
|