mirror of
https://github.com/shareAI-lab/analysis_claude_code.git
synced 2026-09-21 21:03:38 +08:00
refactor: streamline the course to 17 lessons
This commit is contained in:
Haoran
@@ -14,22 +14,27 @@ from unittest.mock import patch
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
LESSON = ROOT / "s15_agent_teams" / "code.py"
|
||||
LESSON = ROOT / "s13_agent_teams" / "code.py"
|
||||
DOWNSTREAM_LESSONS = (
|
||||
ROOT / "s16_mcp_plugin" / "code.py",
|
||||
ROOT / "s17_integrated_harness" / "code.py",
|
||||
ROOT / "s15_integrated_harness" / "code.py",
|
||||
)
|
||||
RUNTIME_LESSONS = (LESSON, *DOWNSTREAM_LESSONS)
|
||||
MCP_LESSONS = (
|
||||
ROOT / "s14_mcp_plugin" / "code.py",
|
||||
ROOT / "s15_integrated_harness" / "code.py",
|
||||
)
|
||||
BACKGROUND_LESSONS = tuple(
|
||||
ROOT / name / "code.py" for name in (
|
||||
"s13_background_tasks",
|
||||
"s14_cron_scheduler",
|
||||
"s15_agent_teams",
|
||||
"s16_mcp_plugin",
|
||||
"s17_integrated_harness",
|
||||
"s11_background_tasks",
|
||||
"s15_integrated_harness",
|
||||
)
|
||||
)
|
||||
CRON_LESSONS = tuple(
|
||||
ROOT / name / "code.py" for name in (
|
||||
"s12_cron_scheduler",
|
||||
"s15_integrated_harness",
|
||||
)
|
||||
)
|
||||
CRON_LESSONS = BACKGROUND_LESSONS[1:]
|
||||
|
||||
|
||||
def load_lesson(temp_cwd: Path, lesson_path: Path = LESSON):
|
||||
@@ -162,6 +167,80 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
self.assertIn("[result] alice: done",
|
||||
lesson.format_team_events(events))
|
||||
|
||||
def test_spawn_claims_the_initial_task_before_starting_the_thread(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp))
|
||||
task = lesson.create_task("Review authentication")
|
||||
schema = next(
|
||||
tool["input_schema"] for tool in lesson.TOOLS
|
||||
if tool["name"] == "spawn_teammate"
|
||||
)
|
||||
self.assertIn("task_id", schema["properties"])
|
||||
|
||||
with patch.object(
|
||||
lesson.threading.Thread, "start", lambda _thread: None
|
||||
):
|
||||
result = lesson.spawn_teammate_thread(
|
||||
"alice", "reviewer", "Review the assigned Task.", task.id
|
||||
)
|
||||
|
||||
self.assertIn(task.id, result)
|
||||
claimed = lesson.load_task(task.id)
|
||||
self.assertEqual(claimed.status, "in_progress")
|
||||
self.assertEqual(claimed.owner, "alice")
|
||||
self.assertEqual(
|
||||
lesson.teammate_assignments["alice"]["task_id"], task.id
|
||||
)
|
||||
|
||||
def test_spawn_allows_an_idle_teammate_without_an_initial_task(self):
|
||||
for lesson_path in RUNTIME_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp), lesson_path)
|
||||
tool_defs = getattr(lesson, "TOOLS", None)
|
||||
if tool_defs is None:
|
||||
tool_defs = lesson.BUILTIN_TOOLS
|
||||
schema = next(
|
||||
tool["input_schema"] for tool in tool_defs
|
||||
if tool["name"] == "spawn_teammate"
|
||||
)
|
||||
self.assertNotIn("task_id", schema["required"])
|
||||
|
||||
with patch.object(
|
||||
lesson.threading.Thread, "start", lambda _thread: None
|
||||
):
|
||||
result = lesson.run_spawn_teammate(
|
||||
"alice", "reviewer", "Wait for a ready Task."
|
||||
)
|
||||
|
||||
self.assertIn("without an initial Task", result)
|
||||
self.assertNotIn("alice", lesson.teammate_assignments)
|
||||
|
||||
def test_teammate_workspace_tools_require_a_claimed_task(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
lesson = load_lesson(root)
|
||||
runtime = lesson.TeammateRuntime(
|
||||
"alice", "reviewer", "Inspect the project.", None, False
|
||||
)
|
||||
|
||||
result = runtime.write("unassigned.txt", "must not be written")
|
||||
|
||||
self.assertIn("Claim a Task", result)
|
||||
self.assertFalse((root / "unassigned.txt").exists())
|
||||
|
||||
def test_plain_message_does_not_change_assignment_or_plan_version(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp))
|
||||
lesson.active_teammates["alice"] = "working"
|
||||
lesson.plan_gates["alice"] = "approved"
|
||||
lesson.assignment_versions["alice"] = 3
|
||||
|
||||
self.assertIn("Sent", lesson.run_send_message("alice", "Continue."))
|
||||
|
||||
self.assertEqual(lesson.plan_gates["alice"], "approved")
|
||||
self.assertEqual(lesson.assignment_versions["alice"], 3)
|
||||
|
||||
def test_worktree_removal_is_host_only(self):
|
||||
for lesson_path in RUNTIME_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
@@ -177,35 +256,83 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
self.assertTrue(callable(lesson.remove_worktree))
|
||||
self.assertFalse(hasattr(lesson, "run_remove_worktree"))
|
||||
|
||||
def test_mcp_lesson_retains_s15_cron_and_background_tools(self):
|
||||
required = {
|
||||
"bash", "schedule_cron", "list_crons", "cancel_cron",
|
||||
"spawn_teammate", "create_worktree",
|
||||
}
|
||||
for lesson_path in RUNTIME_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp), lesson_path)
|
||||
tool_defs = getattr(lesson, "TOOLS", None)
|
||||
if tool_defs is None:
|
||||
tool_defs = lesson.BUILTIN_TOOLS
|
||||
tool_names = {tool["name"] for tool in tool_defs}
|
||||
bash_schema = next(
|
||||
tool["input_schema"] for tool in tool_defs
|
||||
if tool["name"] == "bash"
|
||||
)
|
||||
def test_agent_teams_builds_on_tasks_not_background_or_cron(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
lesson = load_lesson(root)
|
||||
tool_names = {tool["name"] for tool in lesson.TOOLS}
|
||||
|
||||
self.assertTrue(required.issubset(tool_names))
|
||||
self.assertIn(
|
||||
"run_in_background", bash_schema["properties"]
|
||||
)
|
||||
self.assertTrue(
|
||||
lesson.should_run_background(
|
||||
"bash", {"run_in_background": True}
|
||||
)
|
||||
)
|
||||
self.assertTrue(callable(lesson.consume_cron_queue))
|
||||
self.assertTrue(callable(lesson.collect_background_results))
|
||||
self.assertTrue({
|
||||
"bash", "read_file", "write_file", "edit_file", "glob",
|
||||
"create_task", "list_tasks", "get_task", "claim_task",
|
||||
"complete_task", "spawn_teammate", "list_teammates",
|
||||
"send_message", "request_shutdown", "request_plan",
|
||||
"review_plan", "create_worktree",
|
||||
}.issubset(tool_names))
|
||||
self.assertTrue({
|
||||
"schedule_cron", "list_crons", "cancel_cron",
|
||||
}.isdisjoint(tool_names))
|
||||
self.assertNotIn("run_in_background", next(
|
||||
tool["input_schema"] for tool in lesson.TOOLS
|
||||
if tool["name"] == "bash"
|
||||
)["properties"])
|
||||
self.assertFalse((root / ".tasks").exists())
|
||||
self.assertFalse((root / ".mailboxes").exists())
|
||||
self.assertFalse((root / ".worktrees").exists())
|
||||
|
||||
def test_integrated_harness_reuses_memory_recall_and_extraction(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp), DOWNSTREAM_LESSONS[0])
|
||||
calls = []
|
||||
lesson.MEMORY_RUNTIME = types.SimpleNamespace(
|
||||
read_memory_index=lambda: "- [Style](style.md) - Project style",
|
||||
load_memories=lambda messages: (
|
||||
calls.append(("recall", list(messages)))
|
||||
or '[{"source":"style.md","content":"Use black."}]'
|
||||
),
|
||||
extract_memories=lambda messages: (
|
||||
calls.append(("extract", list(messages))) or 1
|
||||
),
|
||||
consolidate_memories=lambda: calls.append(("consolidate", None)),
|
||||
)
|
||||
messages = [{"role": "user", "content": "Format this file."}]
|
||||
|
||||
context = lesson.update_context({}, messages)
|
||||
system = lesson.assemble_system_prompt(context)
|
||||
lesson.remember_after_turn(messages)
|
||||
|
||||
self.assertIn("Memory catalog", system)
|
||||
self.assertIn("Relevant memory records", system)
|
||||
self.assertEqual(
|
||||
[name for name, _payload in calls],
|
||||
["recall", "extract", "consolidate"],
|
||||
)
|
||||
|
||||
def test_mcp_lesson_builds_on_the_base_kernel(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(
|
||||
Path(tmp), ROOT / "s14_mcp_plugin" / "code.py"
|
||||
)
|
||||
tools_before, handlers_before = lesson.assemble_tool_pool()
|
||||
self.assertEqual(
|
||||
{tool["name"] for tool in tools_before},
|
||||
{"bash", "read_file", "write_file", "edit_file", "glob",
|
||||
"connect_mcp"},
|
||||
)
|
||||
self.assertNotIn("mcp__docs__search", handlers_before)
|
||||
|
||||
self.assertIn(
|
||||
"Connected to MCP server 'docs'", lesson.connect_mcp("docs")
|
||||
)
|
||||
tools_after, handlers_after = lesson.assemble_tool_pool()
|
||||
self.assertIn(
|
||||
"mcp__docs__search",
|
||||
{tool["name"] for tool in tools_after},
|
||||
)
|
||||
self.assertEqual(
|
||||
handlers_after["mcp__docs__search"](query="hooks"),
|
||||
"[docs] Found 3 results for 'hooks'",
|
||||
)
|
||||
|
||||
def test_background_dispatch_is_bash_only_and_reports_failures(self):
|
||||
for lesson_path in BACKGROUND_LESSONS:
|
||||
@@ -223,7 +350,7 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
input={"command": "exit 7", "run_in_background": True},
|
||||
)
|
||||
if lesson_path.parent.name in {
|
||||
"s16_mcp_plugin", "s17_integrated_harness"
|
||||
"s14_mcp_plugin", "s15_integrated_harness"
|
||||
}:
|
||||
bg_id = lesson.start_background_task(block, {})
|
||||
else:
|
||||
@@ -331,7 +458,7 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
content=[], stop_reason="end_turn"
|
||||
)
|
||||
messages = []
|
||||
if lesson_path.parent.name == "s17_integrated_harness":
|
||||
if lesson_path.parent.name == "s15_integrated_harness":
|
||||
lesson.agent_loop(messages, {}, "scheduled delivery")
|
||||
else:
|
||||
lesson.agent_loop(messages, {})
|
||||
@@ -364,7 +491,7 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
)
|
||||
|
||||
messages = []
|
||||
if lesson_path.parent.name == "s17_integrated_harness":
|
||||
if lesson_path.parent.name == "s15_integrated_harness":
|
||||
lesson.agent_loop(messages, {}, "scheduled retry")
|
||||
else:
|
||||
lesson.agent_loop(messages, {})
|
||||
@@ -434,41 +561,42 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
self.assertIn("Cancelled", lesson.cancel_job(job.id))
|
||||
self.assertEqual(lesson.consume_cron_queue(), [])
|
||||
|
||||
def test_integrated_permission_uses_host_mcp_policy(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(
|
||||
Path(tmp), ROOT / "s17_integrated_harness" / "code.py"
|
||||
)
|
||||
lesson.connect_mcp("deploy")
|
||||
status = types.SimpleNamespace(
|
||||
name="mcp__deploy__status", input={"service": "web"}
|
||||
)
|
||||
trigger = types.SimpleNamespace(
|
||||
name="mcp__deploy__trigger", input={"service": "web"}
|
||||
)
|
||||
def test_mcp_permission_uses_host_policy(self):
|
||||
for lesson_path in MCP_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp), lesson_path)
|
||||
lesson.connect_mcp("deploy")
|
||||
lesson.assemble_tool_pool()
|
||||
status = types.SimpleNamespace(
|
||||
name="mcp__deploy__status", input={"service": "web"}
|
||||
)
|
||||
trigger = types.SimpleNamespace(
|
||||
name="mcp__deploy__trigger", input={"service": "web"}
|
||||
)
|
||||
|
||||
self.assertIsNone(lesson.permission_hook(status))
|
||||
with patch("builtins.input", return_value="no"):
|
||||
self.assertEqual(
|
||||
lesson.permission_hook(trigger),
|
||||
"Permission denied by user",
|
||||
)
|
||||
self.assertIsNone(lesson.permission_hook(status))
|
||||
with patch("builtins.input", return_value="no"):
|
||||
self.assertEqual(
|
||||
lesson.permission_hook(trigger),
|
||||
"Permission denied by user",
|
||||
)
|
||||
|
||||
spoofed = types.SimpleNamespace(
|
||||
name="mcp__third_party__erase",
|
||||
input={"description": "Erase records. (readOnly)"},
|
||||
)
|
||||
with patch("builtins.input", return_value="no"):
|
||||
self.assertEqual(
|
||||
lesson.permission_hook(spoofed),
|
||||
"Permission denied by user",
|
||||
)
|
||||
spoofed = types.SimpleNamespace(
|
||||
name="mcp__third_party__erase",
|
||||
input={"description": "Erase records. (readOnly)"},
|
||||
)
|
||||
with patch("builtins.input", return_value="no"):
|
||||
self.assertEqual(
|
||||
lesson.permission_hook(spoofed),
|
||||
"Permission denied by user",
|
||||
)
|
||||
|
||||
def test_integrated_permission_requires_approval_for_every_shell_command(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
lesson = load_lesson(
|
||||
root, ROOT / "s17_integrated_harness" / "code.py"
|
||||
root, ROOT / "s15_integrated_harness" / "code.py"
|
||||
)
|
||||
outside = root.parent / f"outside-{time.time_ns()}.txt"
|
||||
block = types.SimpleNamespace(
|
||||
@@ -544,29 +672,95 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
def test_plan_gate_blocks_mutating_tools_until_approval(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp))
|
||||
calls = []
|
||||
block = types.SimpleNamespace(
|
||||
name="write_file",
|
||||
input={"path": "config.py", "content": "VALUE = 1"},
|
||||
)
|
||||
handlers = {
|
||||
"write_file": lambda **kwargs: calls.append(kwargs) or "wrote"
|
||||
cases = {
|
||||
"write_file": {"path": "config.py", "content": "VALUE = 1"},
|
||||
"edit_file": {
|
||||
"path": "config.py", "old_text": "0", "new_text": "1"
|
||||
},
|
||||
}
|
||||
for tool_name, tool_input in cases.items():
|
||||
with self.subTest(tool=tool_name):
|
||||
calls = []
|
||||
block = types.SimpleNamespace(
|
||||
name=tool_name, input=tool_input
|
||||
)
|
||||
handlers = {
|
||||
tool_name: lambda **kwargs: calls.append(kwargs) or "done"
|
||||
}
|
||||
|
||||
lesson.plan_gates["alice"] = "pending"
|
||||
blocked = lesson._run_teammate_tool("alice", block, handlers)
|
||||
self.assertIn("Blocked", blocked)
|
||||
self.assertEqual(calls, [])
|
||||
lesson.plan_gates["alice"] = "pending"
|
||||
blocked = lesson._run_teammate_tool(
|
||||
"alice", block, handlers
|
||||
)
|
||||
self.assertIn("Blocked", blocked)
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
lesson.plan_gates["alice"] = "approved"
|
||||
allowed = lesson._run_teammate_tool("alice", block, handlers)
|
||||
self.assertEqual(allowed, "wrote")
|
||||
self.assertEqual(len(calls), 1)
|
||||
lesson.plan_gates["alice"] = "approved"
|
||||
allowed = lesson._run_teammate_tool(
|
||||
"alice", block, handlers
|
||||
)
|
||||
self.assertEqual(allowed, "done")
|
||||
self.assertEqual(len(calls), 1)
|
||||
|
||||
def test_s17_teammate_dispatch_runs_permission_and_post_hooks(self):
|
||||
def test_teammate_tool_errors_become_tool_results(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp))
|
||||
lesson.plan_gates["alice"] = "not_required"
|
||||
block = types.SimpleNamespace(
|
||||
name="write_file", input={"path": "config.py"}
|
||||
)
|
||||
|
||||
result = lesson._run_teammate_tool(
|
||||
"alice", block,
|
||||
{"write_file": lambda path, content: "wrote"},
|
||||
)
|
||||
|
||||
self.assertIn("TypeError", result)
|
||||
|
||||
def test_teammate_keeps_complete_tool_history(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp))
|
||||
lesson.IDLE_SCAN_INTERVAL = 5.0
|
||||
calls = 0
|
||||
|
||||
def respond(**kwargs):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
self.assertEqual(
|
||||
kwargs["messages"][0]["content"], "Inspect the project."
|
||||
)
|
||||
if calls <= 11:
|
||||
return types.SimpleNamespace(
|
||||
stop_reason="tool_use",
|
||||
content=[types.SimpleNamespace(
|
||||
type="tool_use", name="list_tasks",
|
||||
id=f"list-{calls}", input={},
|
||||
)],
|
||||
)
|
||||
return types.SimpleNamespace(
|
||||
stop_reason="end_turn",
|
||||
content=[types.SimpleNamespace(
|
||||
type="text", text="Inspection complete."
|
||||
)],
|
||||
)
|
||||
|
||||
lesson.client.messages.create = respond
|
||||
lesson.spawn_teammate_thread(
|
||||
"alice", "reviewer", "Inspect the project."
|
||||
)
|
||||
self.assertTrue(wait_until(
|
||||
lambda: lesson.BUS.peek("lead"), timeout=3.0
|
||||
))
|
||||
self.assertEqual(calls, 12)
|
||||
lesson.run_request_shutdown("alice")
|
||||
self.assertTrue(wait_until(
|
||||
lambda: "alice" not in lesson.active_teammates
|
||||
))
|
||||
|
||||
def test_s15_teammate_dispatch_runs_permission_and_post_hooks(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(
|
||||
Path(tmp), ROOT / "s17_integrated_harness" / "code.py"
|
||||
Path(tmp), ROOT / "s15_integrated_harness" / "code.py"
|
||||
)
|
||||
block = types.SimpleNamespace(
|
||||
name="write_file",
|
||||
@@ -608,10 +802,10 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
],
|
||||
)
|
||||
|
||||
def test_s17_teammate_reads_shutdown_between_tool_rounds(self):
|
||||
def test_s15_teammate_reads_shutdown_between_tool_rounds(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(
|
||||
Path(tmp), ROOT / "s17_integrated_harness" / "code.py"
|
||||
Path(tmp), ROOT / "s15_integrated_harness" / "code.py"
|
||||
)
|
||||
entered = threading.Event()
|
||||
release = threading.Event()
|
||||
@@ -640,7 +834,7 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
self.assertEqual(calls, ["llm"])
|
||||
|
||||
def test_normalized_mcp_tool_name_collisions_are_rejected(self):
|
||||
for lesson_path in DOWNSTREAM_LESSONS:
|
||||
for lesson_path in MCP_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp), lesson_path)
|
||||
@@ -931,10 +1125,10 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
self.assertEqual([event["type"] for event in events], ["error"])
|
||||
self.assertIn("simulated dispatch failure", events[0]["content"])
|
||||
|
||||
def test_s17_completed_background_task_wakes_the_agent_once(self):
|
||||
def test_s15_completed_background_task_wakes_the_agent_once(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(
|
||||
Path(tmp), ROOT / "s17_integrated_harness" / "code.py"
|
||||
Path(tmp), ROOT / "s15_integrated_harness" / "code.py"
|
||||
)
|
||||
seen_messages = []
|
||||
|
||||
@@ -1028,7 +1222,7 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
wait_until(lambda: "alice" not in lesson.active_teammates)
|
||||
)
|
||||
|
||||
def test_autonomous_claim_is_atomic_across_teammates(self):
|
||||
def test_idle_claim_is_atomic_across_teammates(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp))
|
||||
task = lesson.create_task("Refactor auth")
|
||||
|
||||
157
tests/test_background_tasks.py
Normal file
157
tests/test_background_tasks.py
Normal file
@@ -0,0 +1,157 @@
|
||||
import copy
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
LESSON = ROOT / "s11_background_tasks" / "code.py"
|
||||
|
||||
|
||||
def load_lesson(workdir: Path):
|
||||
fake_anthropic = types.ModuleType("anthropic")
|
||||
|
||||
class FakeAnthropic:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.messages = types.SimpleNamespace(create=None)
|
||||
|
||||
fake_dotenv = types.ModuleType("dotenv")
|
||||
fake_anthropic.Anthropic = FakeAnthropic
|
||||
fake_dotenv.load_dotenv = lambda override=True: None
|
||||
|
||||
previous_modules = {
|
||||
"anthropic": sys.modules.get("anthropic"),
|
||||
"dotenv": sys.modules.get("dotenv"),
|
||||
}
|
||||
previous_cwd = Path.cwd()
|
||||
previous_model = os.environ.get("MODEL_ID")
|
||||
module_name = f"background_tasks_test_{time.time_ns()}"
|
||||
spec = importlib.util.spec_from_file_location(module_name, LESSON)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
|
||||
sys.modules["anthropic"] = fake_anthropic
|
||||
sys.modules["dotenv"] = fake_dotenv
|
||||
sys.modules[module_name] = module
|
||||
try:
|
||||
os.chdir(workdir)
|
||||
os.environ["MODEL_ID"] = "test-model"
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
finally:
|
||||
os.chdir(previous_cwd)
|
||||
if previous_model is None:
|
||||
os.environ.pop("MODEL_ID", None)
|
||||
else:
|
||||
os.environ["MODEL_ID"] = previous_model
|
||||
for name, previous in previous_modules.items():
|
||||
if previous is None:
|
||||
sys.modules.pop(name, None)
|
||||
else:
|
||||
sys.modules[name] = previous
|
||||
|
||||
|
||||
def wait_until(predicate, timeout: float = 2.0) -> bool:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if predicate():
|
||||
return True
|
||||
time.sleep(0.01)
|
||||
return False
|
||||
|
||||
|
||||
def test_s11_keeps_the_s04_kernel_and_adds_one_bash_option():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp))
|
||||
|
||||
assert {tool["name"] for tool in lesson.TOOLS} == {
|
||||
"bash", "read_file", "write_file", "edit_file", "glob"
|
||||
}
|
||||
bash = next(tool for tool in lesson.TOOLS if tool["name"] == "bash")
|
||||
assert "run_in_background" in bash["input_schema"]["properties"]
|
||||
assert set(lesson.HOOKS) == {
|
||||
"UserPromptSubmit", "PreToolUse", "PostToolUse", "Stop"
|
||||
}
|
||||
assert not hasattr(lesson, "Task")
|
||||
assert not hasattr(lesson, "MEMORY_DIR")
|
||||
|
||||
|
||||
def test_background_execution_requires_an_explicit_bash_flag():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp))
|
||||
|
||||
assert not lesson.should_run_background("bash", {"command": "npm install"})
|
||||
assert lesson.should_run_background(
|
||||
"bash", {"command": "printf ready", "run_in_background": True}
|
||||
)
|
||||
assert not lesson.should_run_background(
|
||||
"write_file", {"run_in_background": True}
|
||||
)
|
||||
|
||||
|
||||
def test_background_bash_passes_permission_before_dispatch():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp))
|
||||
block = types.SimpleNamespace(
|
||||
id="tool_denied",
|
||||
name="bash",
|
||||
input={"command": "rm -rf /tmp/example", "run_in_background": True},
|
||||
type="tool_use",
|
||||
)
|
||||
responses = [
|
||||
types.SimpleNamespace(stop_reason="tool_use", content=[block]),
|
||||
types.SimpleNamespace(
|
||||
stop_reason="end_turn",
|
||||
content=[types.SimpleNamespace(type="text", text="Denied.")],
|
||||
),
|
||||
]
|
||||
lesson.client.messages.create = lambda **_: responses.pop(0)
|
||||
history = [{"role": "user", "content": "Delete the directory"}]
|
||||
|
||||
lesson.agent_loop(history)
|
||||
|
||||
assert not lesson.background_tasks
|
||||
result = history[2]["content"][0]
|
||||
assert result["type"] == "tool_result"
|
||||
assert "Permission denied" in result["content"]
|
||||
|
||||
|
||||
def test_completed_result_is_collected_once_before_a_later_llm_call():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp))
|
||||
block = types.SimpleNamespace(
|
||||
id="tool_ready",
|
||||
name="bash",
|
||||
input={"command": "printf ready", "run_in_background": True},
|
||||
)
|
||||
task_id = lesson.start_background_task(block)
|
||||
assert wait_until(
|
||||
lambda: lesson.background_tasks[task_id]["status"] == "completed"
|
||||
)
|
||||
|
||||
seen_messages = []
|
||||
|
||||
def respond(**kwargs):
|
||||
seen_messages.append(copy.deepcopy(kwargs["messages"]))
|
||||
return types.SimpleNamespace(
|
||||
stop_reason="end_turn",
|
||||
content=[types.SimpleNamespace(type="text", text="Received.")],
|
||||
)
|
||||
|
||||
lesson.client.messages.create = respond
|
||||
history = [{"role": "user", "content": "Continue"}]
|
||||
lesson.agent_loop(history)
|
||||
|
||||
delivered = str(seen_messages[0])
|
||||
assert "<task_notification>" in delivered
|
||||
assert f"<task_id>{task_id}</task_id>" in delivered
|
||||
assert "<status>completed</status>" in delivered
|
||||
assert "ready" in delivered
|
||||
assert lesson.collect_background_results() == []
|
||||
|
||||
|
||||
def test_s11_code_is_ascii():
|
||||
LESSON.read_text(encoding="ascii")
|
||||
@@ -7,7 +7,7 @@ CHAPTERS = sorted(ROOT.glob("s[0-9][0-9]_*"))
|
||||
|
||||
|
||||
def test_every_chapter_uses_english_as_the_default_readme() -> None:
|
||||
assert len(CHAPTERS) == 19
|
||||
assert len(CHAPTERS) == 17
|
||||
|
||||
for chapter in CHAPTERS:
|
||||
assert (chapter / "README.md").is_file()
|
||||
|
||||
@@ -10,8 +10,7 @@ from pathlib import Path
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
MODULES = {
|
||||
"s08": REPO_ROOT / "s08_context_compact" / "code.py",
|
||||
"s09": REPO_ROOT / "s09_memory" / "code.py",
|
||||
"s17": REPO_ROOT / "s17_integrated_harness" / "code.py",
|
||||
"s15": REPO_ROOT / "s15_integrated_harness" / "code.py",
|
||||
}
|
||||
|
||||
|
||||
@@ -107,6 +106,11 @@ def assert_no_orphan_tool_results(testcase, messages):
|
||||
testcase.assertTrue(message_has_tool_use(messages[idx - 1]), messages)
|
||||
|
||||
|
||||
def compaction_api(module):
|
||||
"""Return the chapter's compaction implementation."""
|
||||
return getattr(module, "COMPACTOR", module)
|
||||
|
||||
|
||||
class CompactionToolPairTests(unittest.TestCase):
|
||||
def test_snip_compact_keeps_head_tool_pair(self):
|
||||
messages = [
|
||||
@@ -125,10 +129,9 @@ class CompactionToolPairTests(unittest.TestCase):
|
||||
for name, path in MODULES.items():
|
||||
with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp:
|
||||
module = load_module(f"{name}_head_under_test", path, Path(tmp))
|
||||
if name == "s09":
|
||||
compacted = module.snip_compact(list(messages), mx=6)
|
||||
else:
|
||||
compacted = module.snip_compact(list(messages), max_messages=6)
|
||||
compacted = compaction_api(module).snip_compact(
|
||||
list(messages), max_messages=6
|
||||
)
|
||||
self.assertEqual(compacted[2], messages[2])
|
||||
self.assertEqual(compacted[3], messages[3])
|
||||
assert_no_orphan_tool_results(self, compacted)
|
||||
@@ -150,10 +153,9 @@ class CompactionToolPairTests(unittest.TestCase):
|
||||
for name, path in MODULES.items():
|
||||
with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp:
|
||||
module = load_module(f"{name}_under_test", path, Path(tmp))
|
||||
if name == "s09":
|
||||
compacted = module.snip_compact(list(messages), mx=6)
|
||||
else:
|
||||
compacted = module.snip_compact(list(messages), max_messages=6)
|
||||
compacted = compaction_api(module).snip_compact(
|
||||
list(messages), max_messages=6
|
||||
)
|
||||
assert_no_orphan_tool_results(self, compacted)
|
||||
|
||||
def test_reactive_compact_keeps_tail_tool_pair(self):
|
||||
@@ -172,9 +174,10 @@ class CompactionToolPairTests(unittest.TestCase):
|
||||
for name, path in MODULES.items():
|
||||
with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp:
|
||||
module = load_module(f"{name}_reactive_under_test", path, Path(tmp))
|
||||
module.write_transcript = lambda _messages: Path("transcript.jsonl")
|
||||
module.summarize_history = lambda _messages: "summary"
|
||||
compacted = module.reactive_compact(list(messages), "continue")
|
||||
api = compaction_api(module)
|
||||
api.write_transcript = lambda _messages: Path("transcript.jsonl")
|
||||
api.summarize_history = lambda _messages: "summary"
|
||||
compacted = api.reactive_compact(list(messages), "continue")
|
||||
self.assertEqual(compacted[1], messages[3])
|
||||
assert_no_orphan_tool_results(self, compacted)
|
||||
|
||||
@@ -194,15 +197,16 @@ class CompactionToolPairTests(unittest.TestCase):
|
||||
for name, path in MODULES.items():
|
||||
with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp:
|
||||
module = load_module(f"{name}_reactive_oldhist_under_test", path, Path(tmp))
|
||||
module.write_transcript = lambda _messages: Path("transcript.jsonl")
|
||||
api = compaction_api(module)
|
||||
api.write_transcript = lambda _messages: Path("transcript.jsonl")
|
||||
captured = {}
|
||||
|
||||
def fake_summarize(passed, _store=captured):
|
||||
_store["messages"] = list(passed)
|
||||
return "summary"
|
||||
|
||||
module.summarize_history = fake_summarize
|
||||
compacted = module.reactive_compact(list(messages), "continue")
|
||||
api.summarize_history = fake_summarize
|
||||
compacted = api.reactive_compact(list(messages), "continue")
|
||||
# The summary must cover only the old history, not the kept tail.
|
||||
self.assertEqual(captured["messages"], messages[:4])
|
||||
# The recent tail is appended verbatim after the summary message.
|
||||
@@ -229,24 +233,25 @@ class CompactionToolPairTests(unittest.TestCase):
|
||||
for name, path in MODULES.items():
|
||||
with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp:
|
||||
module = load_module(f"{name}_reactive_pairscope_under_test", path, Path(tmp))
|
||||
module.write_transcript = lambda _messages: Path("transcript.jsonl")
|
||||
api = compaction_api(module)
|
||||
api.write_transcript = lambda _messages: Path("transcript.jsonl")
|
||||
captured = {}
|
||||
|
||||
def fake_summarize(passed, _store=captured):
|
||||
_store["messages"] = list(passed)
|
||||
return "summary"
|
||||
|
||||
module.summarize_history = fake_summarize
|
||||
compacted = module.reactive_compact(list(messages), "continue")
|
||||
api.summarize_history = fake_summarize
|
||||
compacted = api.reactive_compact(list(messages), "continue")
|
||||
# tail_start starts at 4, decrements to 3 to keep the pair intact.
|
||||
self.assertEqual(captured["messages"], messages[:3])
|
||||
self.assertEqual(compacted[1], messages[3])
|
||||
self.assertEqual(compacted[1:], messages[3:])
|
||||
assert_no_orphan_tool_results(self, compacted)
|
||||
|
||||
def test_s17_has_tool_use_still_accepts_content_blocks(self):
|
||||
def test_s15_has_tool_use_still_accepts_content_blocks(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
module = load_module("s17_has_tool_use_under_test", MODULES["s17"], Path(tmp))
|
||||
module = load_module("s15_has_tool_use_under_test", MODULES["s15"], Path(tmp))
|
||||
self.assertTrue(module.has_tool_use([types.SimpleNamespace(type="tool_use")]))
|
||||
self.assertFalse(module.has_tool_use([types.SimpleNamespace(type="text")]))
|
||||
|
||||
|
||||
190
tests/test_cron_scheduler.py
Normal file
190
tests/test_cron_scheduler.py
Normal file
@@ -0,0 +1,190 @@
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import types
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
LESSON = ROOT / "s12_cron_scheduler" / "code.py"
|
||||
|
||||
|
||||
def load_lesson(workdir: Path):
|
||||
fake_anthropic = types.ModuleType("anthropic")
|
||||
fake_dotenv = types.ModuleType("dotenv")
|
||||
|
||||
class FakeAnthropic:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.messages = types.SimpleNamespace(create=None)
|
||||
|
||||
fake_anthropic.Anthropic = FakeAnthropic
|
||||
fake_dotenv.load_dotenv = lambda override=True: None
|
||||
|
||||
previous_modules = {
|
||||
"anthropic": sys.modules.get("anthropic"),
|
||||
"dotenv": sys.modules.get("dotenv"),
|
||||
}
|
||||
previous_cwd = Path.cwd()
|
||||
previous_model = os.environ.get("MODEL_ID")
|
||||
module_name = f"cron_scheduler_test_{time.time_ns()}"
|
||||
spec = importlib.util.spec_from_file_location(module_name, LESSON)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
|
||||
sys.modules["anthropic"] = fake_anthropic
|
||||
sys.modules["dotenv"] = fake_dotenv
|
||||
sys.modules[module_name] = module
|
||||
try:
|
||||
os.chdir(workdir)
|
||||
os.environ["MODEL_ID"] = "test-model"
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
finally:
|
||||
os.chdir(previous_cwd)
|
||||
if previous_model is None:
|
||||
os.environ.pop("MODEL_ID", None)
|
||||
else:
|
||||
os.environ["MODEL_ID"] = previous_model
|
||||
for name, previous in previous_modules.items():
|
||||
if previous is None:
|
||||
sys.modules.pop(name, None)
|
||||
else:
|
||||
sys.modules[name] = previous
|
||||
|
||||
|
||||
def test_s12_keeps_the_s04_kernel_and_adds_three_cron_tools():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp))
|
||||
|
||||
assert [tool["name"] for tool in lesson.TOOLS] == [
|
||||
"bash",
|
||||
"read_file",
|
||||
"write_file",
|
||||
"edit_file",
|
||||
"glob",
|
||||
"schedule_cron",
|
||||
"list_crons",
|
||||
"cancel_cron",
|
||||
]
|
||||
assert set(lesson.HOOKS) == {
|
||||
"UserPromptSubmit", "PreToolUse", "PostToolUse", "Stop"
|
||||
}
|
||||
assert not hasattr(lesson, "Task")
|
||||
assert not hasattr(lesson, "MEMORY_DIR")
|
||||
assert not hasattr(lesson, "background_tasks")
|
||||
|
||||
|
||||
def test_import_does_not_start_runtime_threads():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp))
|
||||
|
||||
assert not lesson.runtime_started
|
||||
assert lesson.runtime_threads == []
|
||||
assert not any(
|
||||
thread.name in {"cron-scheduler", "cron-queue-processor"}
|
||||
for thread in threading.enumerate()
|
||||
)
|
||||
|
||||
|
||||
def test_cron_validation_and_matching():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp))
|
||||
monday_at_nine = datetime(2026, 8, 10, 9, 0)
|
||||
|
||||
assert lesson.validate_cron("0 9 * * 1-5") is None
|
||||
assert lesson.cron_matches("0 9 * * 1-5", monday_at_nine)
|
||||
assert not lesson.cron_matches("30 9 * * 1-5", monday_at_nine)
|
||||
assert "hour" in lesson.validate_cron("0 24 * * *")
|
||||
assert "Expected 5 fields" in lesson.validate_cron("0 9 * *")
|
||||
|
||||
|
||||
def test_schedule_retries_id_collisions_and_rolls_back_failed_persistence(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp))
|
||||
values = iter(["deadbeef", "deadbeef", "cafebabe", "bad0cafe"])
|
||||
monkeypatch.setattr(lesson.secrets, "token_hex", lambda _size: next(values))
|
||||
|
||||
first = lesson.schedule_job("0 9 * * *", "first", durable=False)
|
||||
second = lesson.schedule_job("0 10 * * *", "second", durable=False)
|
||||
assert first.id == "cron_deadbeef"
|
||||
assert second.id == "cron_cafebabe"
|
||||
|
||||
monkeypatch.setattr(
|
||||
lesson,
|
||||
"save_durable_jobs",
|
||||
lambda: (_ for _ in ()).throw(OSError("disk full")),
|
||||
)
|
||||
with pytest.raises(OSError, match="disk full"):
|
||||
lesson.schedule_job("0 11 * * *", "third", durable=True)
|
||||
assert "cron_bad0cafe" not in lesson.scheduled_jobs
|
||||
|
||||
|
||||
def test_failed_model_call_restores_delivery_without_duplicate_message():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp))
|
||||
job = lesson.CronJob(
|
||||
id="cron_retry",
|
||||
cron="* * * * *",
|
||||
prompt="retry the report",
|
||||
recurring=False,
|
||||
durable=True,
|
||||
pending_delivery=True,
|
||||
)
|
||||
lesson.scheduled_jobs[job.id] = job
|
||||
lesson.cron_queue.append(job)
|
||||
lesson.save_durable_jobs()
|
||||
lesson.client.messages.create = (
|
||||
lambda **_: (_ for _ in ()).throw(RuntimeError("offline"))
|
||||
)
|
||||
|
||||
messages = []
|
||||
lesson.agent_loop(messages)
|
||||
|
||||
assert messages == []
|
||||
assert [queued.id for queued in lesson.cron_queue] == [job.id]
|
||||
assert job.id in lesson.scheduled_jobs
|
||||
|
||||
|
||||
def test_scheduled_turn_never_reads_interactive_permission_input():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp))
|
||||
block = types.SimpleNamespace(
|
||||
name="bash",
|
||||
input={"command": "rm build.log"},
|
||||
)
|
||||
results = []
|
||||
|
||||
with patch("builtins.input", side_effect=AssertionError("input called")):
|
||||
thread = threading.Thread(
|
||||
target=lambda: results.append(lesson.permission_hook(block))
|
||||
)
|
||||
thread.start()
|
||||
thread.join(timeout=1)
|
||||
|
||||
assert results == [
|
||||
"Permission denied: scheduled turns cannot request interactive approval"
|
||||
]
|
||||
|
||||
|
||||
def test_corrupt_durable_store_reports_an_error(capsys: pytest.CaptureFixture[str]):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp))
|
||||
lesson.DURABLE_PATH.write_text("{broken")
|
||||
|
||||
lesson.load_durable_jobs()
|
||||
|
||||
assert "could not load .scheduled_tasks.json" in capsys.readouterr().out
|
||||
assert lesson.scheduled_jobs == {}
|
||||
|
||||
|
||||
def test_s12_code_is_ascii():
|
||||
LESSON.read_text(encoding="ascii")
|
||||
@@ -9,8 +9,8 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
MODULE_PATH = REPO_ROOT / "s19_goal_loop" / "code.py"
|
||||
MODULE_NAME = "s19_goal_loop_under_test"
|
||||
MODULE_PATH = REPO_ROOT / "s17_goal_loop" / "code.py"
|
||||
MODULE_NAME = "s17_goal_loop_under_test"
|
||||
SPEC = importlib.util.spec_from_file_location(MODULE_NAME, MODULE_PATH)
|
||||
if SPEC is None or SPEC.loader is None:
|
||||
raise RuntimeError(f"Unable to load {MODULE_PATH}")
|
||||
@@ -465,3 +465,57 @@ def test_transcript_trimming_keeps_complete_recent_messages() -> None:
|
||||
|
||||
assert "recent result" in rendered
|
||||
assert "old-" not in rendered
|
||||
|
||||
|
||||
def test_transcript_trims_the_middle_of_one_oversized_message() -> None:
|
||||
rendered = goal_loop.transcript_text(
|
||||
[{"role": "user", "content": "START" + "x" * 100 + "END"}],
|
||||
max_characters=40,
|
||||
)
|
||||
|
||||
assert len(rendered) == 40
|
||||
assert rendered.startswith("USER:\nSTART")
|
||||
assert rendered.endswith("END")
|
||||
assert "middle omitted" in rendered
|
||||
|
||||
|
||||
def test_goal_loop_keeps_the_s04_base_tools_and_permission_hook(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
controller = goal_loop.GoalController(RecordingEvaluator())
|
||||
session = goal_loop.AgentSession(
|
||||
client=FakeClient([]),
|
||||
model="worker-model",
|
||||
goal=controller,
|
||||
workdir=tmp_path,
|
||||
)
|
||||
|
||||
assert {tool["name"] for tool in goal_loop.TOOLS} == {
|
||||
"bash", "read_file", "write_file", "edit_file", "glob"
|
||||
}
|
||||
block = SimpleNamespace(
|
||||
name="write_file",
|
||||
input={"path": "../outside.txt", "content": "blocked"},
|
||||
)
|
||||
assert "outside" in session.trigger_hooks("PreToolUse", block)
|
||||
assert not (tmp_path.parent / "outside.txt").exists()
|
||||
|
||||
|
||||
def test_goal_loop_file_tools_use_the_current_repository(tmp_path: Path) -> None:
|
||||
controller = goal_loop.GoalController(RecordingEvaluator())
|
||||
session = goal_loop.AgentSession(
|
||||
client=FakeClient([]),
|
||||
model="worker-model",
|
||||
goal=controller,
|
||||
workdir=tmp_path,
|
||||
)
|
||||
|
||||
assert "Wrote" in session._run_tool(
|
||||
"write_file", {"path": "src/value.txt", "content": "old"}
|
||||
)
|
||||
assert "Edited" in session._run_tool(
|
||||
"edit_file",
|
||||
{"path": "src/value.txt", "old_text": "old", "new_text": "new"},
|
||||
)
|
||||
assert session._run_tool("glob", {"pattern": "src/*.txt"}) == "src/value.txt"
|
||||
assert (tmp_path / "src" / "value.txt").read_text() == "new"
|
||||
|
||||
95
tests/test_skill_loading.py
Normal file
95
tests/test_skill_loading.py
Normal file
@@ -0,0 +1,95 @@
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
LESSON = ROOT / "s07_skill_loading" / "code.py"
|
||||
|
||||
|
||||
def load_lesson(workdir: Path):
|
||||
fake_anthropic = types.ModuleType("anthropic")
|
||||
fake_dotenv = types.ModuleType("dotenv")
|
||||
|
||||
class FakeAnthropic:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.messages = types.SimpleNamespace(create=None)
|
||||
|
||||
fake_anthropic.Anthropic = FakeAnthropic
|
||||
fake_dotenv.load_dotenv = lambda override=True: None
|
||||
|
||||
previous_modules = {
|
||||
"anthropic": sys.modules.get("anthropic"),
|
||||
"dotenv": sys.modules.get("dotenv"),
|
||||
}
|
||||
previous_cwd = Path.cwd()
|
||||
previous_model = os.environ.get("MODEL_ID")
|
||||
|
||||
spec = importlib.util.spec_from_file_location("s07_skill_test", LESSON)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
|
||||
sys.modules["anthropic"] = fake_anthropic
|
||||
sys.modules["dotenv"] = fake_dotenv
|
||||
try:
|
||||
os.chdir(workdir)
|
||||
os.environ["MODEL_ID"] = "test-model"
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
finally:
|
||||
os.chdir(previous_cwd)
|
||||
if previous_model is None:
|
||||
os.environ.pop("MODEL_ID", None)
|
||||
else:
|
||||
os.environ["MODEL_ID"] = previous_model
|
||||
for name, previous in previous_modules.items():
|
||||
if previous is None:
|
||||
sys.modules.pop(name, None)
|
||||
else:
|
||||
sys.modules[name] = previous
|
||||
|
||||
|
||||
def test_catalog_stays_small_and_load_skill_returns_the_full_file() -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
skill_dir = root / "skills" / "code-review"
|
||||
skill_dir.mkdir(parents=True)
|
||||
manifest = """---
|
||||
name: code-review
|
||||
description: |
|
||||
Review code for bugs,
|
||||
regressions, and missing tests.
|
||||
---
|
||||
|
||||
# Code Review
|
||||
|
||||
UNIQUE_FULL_INSTRUCTION
|
||||
"""
|
||||
(skill_dir / "SKILL.md").write_text(manifest)
|
||||
|
||||
lesson = load_lesson(root)
|
||||
|
||||
assert lesson.SKILL_LOADER.catalog() == (
|
||||
"- code-review: Review code for bugs, regressions, and missing tests."
|
||||
)
|
||||
assert "code-review" in lesson.SYSTEM
|
||||
assert "UNIQUE_FULL_INSTRUCTION" not in lesson.SYSTEM
|
||||
assert lesson.SKILL_LOADER.load("code-review") == manifest
|
||||
assert lesson.TOOL_HANDLERS["load_skill"]("code-review") == manifest
|
||||
|
||||
|
||||
def test_s07_exposes_only_base_tools_and_load_skill() -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp))
|
||||
|
||||
assert [tool["name"] for tool in lesson.TOOLS] == [
|
||||
"bash",
|
||||
"read_file",
|
||||
"write_file",
|
||||
"edit_file",
|
||||
"glob",
|
||||
"load_skill",
|
||||
]
|
||||
162
tests/test_task_system.py
Normal file
162
tests/test_task_system.py
Normal file
@@ -0,0 +1,162 @@
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
LESSON = ROOT / "s10_task_system" / "code.py"
|
||||
|
||||
|
||||
def load_lesson(workdir: Path):
|
||||
fake_anthropic = types.ModuleType("anthropic")
|
||||
fake_dotenv = types.ModuleType("dotenv")
|
||||
|
||||
class FakeAnthropic:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.messages = types.SimpleNamespace(create=None)
|
||||
|
||||
fake_anthropic.Anthropic = FakeAnthropic
|
||||
fake_dotenv.load_dotenv = lambda override=True: None
|
||||
|
||||
previous_modules = {
|
||||
"anthropic": sys.modules.get("anthropic"),
|
||||
"dotenv": sys.modules.get("dotenv"),
|
||||
}
|
||||
previous_cwd = Path.cwd()
|
||||
previous_model = os.environ.get("MODEL_ID")
|
||||
|
||||
module_name = f"s10_task_system_test_{id(workdir)}"
|
||||
spec = importlib.util.spec_from_file_location(module_name, LESSON)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
|
||||
sys.modules["anthropic"] = fake_anthropic
|
||||
sys.modules["dotenv"] = fake_dotenv
|
||||
sys.modules[module_name] = module
|
||||
try:
|
||||
os.chdir(workdir)
|
||||
os.environ["MODEL_ID"] = "test-model"
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
finally:
|
||||
os.chdir(previous_cwd)
|
||||
sys.modules.pop(module_name, None)
|
||||
if previous_model is None:
|
||||
os.environ.pop("MODEL_ID", None)
|
||||
else:
|
||||
os.environ["MODEL_ID"] = previous_model
|
||||
for name, previous in previous_modules.items():
|
||||
if previous is None:
|
||||
sys.modules.pop(name, None)
|
||||
else:
|
||||
sys.modules[name] = previous
|
||||
|
||||
|
||||
def tool_call(name: str, **arguments):
|
||||
return types.SimpleNamespace(name=name, input=arguments, id="tool-1")
|
||||
|
||||
|
||||
def test_s10_keeps_the_s04_kernel_and_adds_task_tools() -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
workdir = Path(tmp)
|
||||
lesson = load_lesson(workdir)
|
||||
|
||||
assert [tool["name"] for tool in lesson.TOOLS] == [
|
||||
"bash",
|
||||
"read_file",
|
||||
"write_file",
|
||||
"edit_file",
|
||||
"glob",
|
||||
"create_task",
|
||||
"list_tasks",
|
||||
"get_task",
|
||||
"claim_task",
|
||||
"complete_task",
|
||||
]
|
||||
assert lesson.permission_hook in lesson.HOOKS["PreToolUse"]
|
||||
assert hasattr(lesson, "execute_tool")
|
||||
assert not hasattr(lesson, "MEMORY_DIR")
|
||||
assert not (workdir / ".tasks").exists()
|
||||
|
||||
|
||||
def test_dependencies_gate_claim_and_completion_checks_owner() -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
workdir = Path(tmp)
|
||||
lesson = load_lesson(workdir)
|
||||
|
||||
schema = lesson.create_task("create schema")
|
||||
api = lesson.create_task("write API", blockedBy=[schema.id])
|
||||
|
||||
assert lesson.claim_task(api.id) == f"Blocked by: ['{schema.id}']"
|
||||
assert "Claimed" in lesson.claim_task(schema.id)
|
||||
assert "Unblocked: write API" in lesson.complete_task(schema.id)
|
||||
assert "Claimed" in lesson.claim_task(api.id)
|
||||
assert "owned by agent, not other" in lesson.complete_task(
|
||||
api.id, owner="other"
|
||||
)
|
||||
assert "Completed" in lesson.complete_task(api.id)
|
||||
assert lesson.load_task(api.id).status == "completed"
|
||||
|
||||
|
||||
def test_invalid_and_missing_task_ids_become_tool_results() -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp))
|
||||
|
||||
invalid = lesson.execute_tool(tool_call("get_task", task_id="../outside"))
|
||||
missing = lesson.execute_tool(
|
||||
tool_call("claim_task", task_id="task_00000000")
|
||||
)
|
||||
|
||||
assert invalid.startswith("Error: Invalid task ID")
|
||||
assert missing.startswith("Error:")
|
||||
|
||||
|
||||
def test_create_retries_instead_of_overwriting_an_existing_id(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp))
|
||||
values = iter(["deadbeef", "deadbeef", "cafebabe"])
|
||||
monkeypatch.setattr(lesson.secrets, "token_hex", lambda _size: next(values))
|
||||
|
||||
first = lesson.create_task("first")
|
||||
second = lesson.create_task("second")
|
||||
|
||||
assert first.id == "task_deadbeef"
|
||||
assert second.id == "task_cafebabe"
|
||||
assert [task.subject for task in lesson.list_tasks()] == ["second", "first"]
|
||||
|
||||
|
||||
def test_create_rejects_unknown_dependencies() -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp))
|
||||
|
||||
output = lesson.execute_tool(tool_call(
|
||||
"create_task",
|
||||
subject="write API",
|
||||
blockedBy=["task_00000000"],
|
||||
))
|
||||
|
||||
assert output == "Error: Dependency not found: task_00000000"
|
||||
|
||||
|
||||
def test_task_store_rejects_a_symlink_outside_the_workspace() -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with tempfile.TemporaryDirectory() as outside:
|
||||
workdir = Path(tmp)
|
||||
(workdir / ".tasks").symlink_to(
|
||||
Path(outside), target_is_directory=True
|
||||
)
|
||||
lesson = load_lesson(workdir)
|
||||
|
||||
output = lesson.execute_tool(
|
||||
tool_call("create_task", subject="unsafe")
|
||||
)
|
||||
|
||||
assert output == "Error: Task store escapes the workspace"
|
||||
assert list(Path(outside).iterdir()) == []
|
||||
@@ -10,9 +10,7 @@ from pathlib import Path
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
COURSE_MODULES = [
|
||||
("s05", REPO_ROOT / "s05_todo_write" / "code.py"),
|
||||
("s07", REPO_ROOT / "s07_skill_loading" / "code.py"),
|
||||
("s08", REPO_ROOT / "s08_context_compact" / "code.py"),
|
||||
("s17", REPO_ROOT / "s17_integrated_harness" / "code.py"),
|
||||
("s15", REPO_ROOT / "s15_integrated_harness" / "code.py"),
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -25,8 +25,8 @@ def load_lesson(name: str, script: Path):
|
||||
return module
|
||||
|
||||
|
||||
def test_s15_scenario_uses_the_real_plan_protocol() -> None:
|
||||
steps = load_scenario("s15")["steps"]
|
||||
def test_s13_scenario_uses_the_real_plan_protocol() -> None:
|
||||
steps = load_scenario("s13")["steps"]
|
||||
spawn = next(
|
||||
step for step in steps
|
||||
if step.get("toolName") == "spawn_teammate"
|
||||
@@ -34,7 +34,7 @@ def test_s15_scenario_uses_the_real_plan_protocol() -> None:
|
||||
)
|
||||
claim_index = next(
|
||||
index for index, step in enumerate(steps)
|
||||
if "claim_next_task(backend)" in step.get("content", "")
|
||||
if "spawn_teammate(backend" in step.get("content", "")
|
||||
)
|
||||
request_index = next(
|
||||
index for index, step in enumerate(steps)
|
||||
@@ -50,7 +50,9 @@ def test_s15_scenario_uses_the_real_plan_protocol() -> None:
|
||||
)
|
||||
|
||||
review = json.loads(steps[review_index]["content"])
|
||||
assert json.loads(spawn["content"])["require_plan"] is True
|
||||
spawn_input = json.loads(spawn["content"])
|
||||
assert spawn_input["require_plan"] is True
|
||||
assert re.fullmatch(r"task_[0-9a-f]{8}", spawn_input["task_id"])
|
||||
assert claim_index < request_index < review_index < response_index
|
||||
assert review["request_id"] == "req_000007"
|
||||
assert re.fullmatch(r"req_\d{6}", review["request_id"])
|
||||
@@ -58,8 +60,8 @@ def test_s15_scenario_uses_the_real_plan_protocol() -> None:
|
||||
assert "approved" not in review
|
||||
|
||||
|
||||
def test_s17_scenario_calls_the_discovered_mcp_tool() -> None:
|
||||
steps = load_scenario("s17")["steps"]
|
||||
def test_s15_scenario_calls_the_discovered_mcp_tool() -> None:
|
||||
steps = load_scenario("s15")["steps"]
|
||||
bash_index = next(
|
||||
index for index, step in enumerate(steps)
|
||||
if step.get("toolName") == "bash"
|
||||
@@ -96,13 +98,13 @@ def test_s17_scenario_calls_the_discovered_mcp_tool() -> None:
|
||||
assert connect_index < status_index < result_index
|
||||
|
||||
|
||||
def test_s17_runtime_discovers_and_dispatches_mcp_tools(
|
||||
def test_s15_runtime_discovers_and_dispatches_mcp_tools(
|
||||
tmp_path: Path, monkeypatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("MODEL_ID", "test-model")
|
||||
harness = load_lesson(
|
||||
"integrated_mcp_scenario_test",
|
||||
ROOT / "s17_integrated_harness" / "code.py",
|
||||
ROOT / "s15_integrated_harness" / "code.py",
|
||||
)
|
||||
harness.WORKDIR = tmp_path
|
||||
|
||||
@@ -117,8 +119,8 @@ def test_s17_runtime_discovers_and_dispatches_mcp_tools(
|
||||
)
|
||||
|
||||
|
||||
def test_s18_scenario_matches_the_deterministic_runtime(tmp_path: Path) -> None:
|
||||
scenario = load_scenario("s18")
|
||||
def test_s16_scenario_matches_the_deterministic_runtime(tmp_path: Path) -> None:
|
||||
scenario = load_scenario("s16")
|
||||
workflow_call = next(
|
||||
step for step in scenario["steps"]
|
||||
if step.get("toolName") == "Workflow" and step["type"] == "tool_call"
|
||||
@@ -131,7 +133,7 @@ def test_s18_scenario_matches_the_deterministic_runtime(tmp_path: Path) -> None:
|
||||
shown_result = json.loads(workflow_result["content"])
|
||||
|
||||
workflow = load_lesson(
|
||||
"workflow_scenario_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
"workflow_scenario_test", ROOT / "s16_workflow_runtime" / "code.py"
|
||||
)
|
||||
workflow.STORE = tmp_path
|
||||
workflow.create_run_id = lambda _meta: "wf_review-changes_0000000000001a7b"
|
||||
@@ -141,26 +143,26 @@ def test_s18_scenario_matches_the_deterministic_runtime(tmp_path: Path) -> None:
|
||||
assert shown_result == actual
|
||||
|
||||
|
||||
def test_generated_s18_metadata_extends_s17_without_registry_false_positives() -> None:
|
||||
def test_generated_s16_metadata_extends_s15_without_registry_false_positives() -> None:
|
||||
versions = json.loads(GENERATED_VERSIONS.read_text())
|
||||
by_id = {version["id"]: version for version in versions["versions"]}
|
||||
s17 = by_id["s17"]
|
||||
s18 = by_id["s18"]
|
||||
s15 = by_id["s15"]
|
||||
s16 = by_id["s16"]
|
||||
|
||||
assert set(s17["tools"]) < set(s18["tools"])
|
||||
assert s18["newTools"] == ["Workflow"]
|
||||
assert "Workflow" in s18["tools"]
|
||||
assert "review-changes" not in s18["tools"]
|
||||
assert set(s15["tools"]) < set(s16["tools"])
|
||||
assert s16["newTools"] == ["Workflow"]
|
||||
assert "Workflow" in s16["tools"]
|
||||
assert "review-changes" not in s16["tools"]
|
||||
chapter_dirs = {
|
||||
path.name.split("_", 1)[0]: path
|
||||
for path in ROOT.glob("s[0-9][0-9]_*")
|
||||
}
|
||||
for lesson_id in ("s13", "s14", "s15", "s16", "s17", "s18"):
|
||||
for lesson_id in ("s11", "s12", "s13", "s14", "s15", "s16"):
|
||||
assert by_id[lesson_id]["source"] == (
|
||||
chapter_dirs[lesson_id] / "code.py"
|
||||
).read_text()
|
||||
signatures = {
|
||||
function["name"]: function["signature"]
|
||||
for function in s18["functions"]
|
||||
for function in s16["functions"]
|
||||
}
|
||||
assert signatures["run_workflow"].startswith("async def run_workflow(")
|
||||
|
||||
@@ -7,6 +7,7 @@ import multiprocessing
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
@@ -51,7 +52,7 @@ def run_lesson(script: Path, *args: str) -> str:
|
||||
|
||||
def test_workflow_runtime_resumes_from_journal(tmp_path: Path) -> None:
|
||||
script = tmp_path / "code.py"
|
||||
shutil.copy2(ROOT / "s18_workflow_runtime" / "code.py", script)
|
||||
shutil.copy2(ROOT / "s16_workflow_runtime" / "code.py", script)
|
||||
|
||||
first = run_lesson(script, "demo")
|
||||
resumed = run_lesson(script, "resume")
|
||||
@@ -64,19 +65,26 @@ def test_workflow_runtime_resumes_from_journal(tmp_path: Path) -> None:
|
||||
|
||||
def test_workflow_runtime_rejects_unsafe_artifact_names() -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_name_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
"workflow_name_test", ROOT / "s16_workflow_runtime" / "code.py"
|
||||
)
|
||||
|
||||
for name in ("../escape", "../../escape", "nested/name"):
|
||||
with pytest.raises(workflow.WorkflowInputError):
|
||||
workflow.validate_meta({"name": name, "description": "unsafe"})
|
||||
|
||||
severity = workflow.FINDINGS_SCHEMA["properties"]["findings"]["items"][
|
||||
"properties"
|
||||
]["severity"]
|
||||
validator = workflow.SimpleJsonSchema(severity)
|
||||
assert validator.validate("high") == (True, None)
|
||||
assert validator.validate("warning")[0] is False
|
||||
|
||||
|
||||
def test_workflow_runtime_enforces_budget_and_shared_agent_cap(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_limit_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
"workflow_limit_test", ROOT / "s16_workflow_runtime" / "code.py"
|
||||
)
|
||||
budget = workflow.Budget(total=1)
|
||||
with pytest.raises(workflow.WorkflowInputError):
|
||||
@@ -120,7 +128,7 @@ def test_workflow_runtime_enforces_budget_and_shared_agent_cap(
|
||||
|
||||
def test_workflow_runtime_rejects_corrupt_resume_journal(tmp_path: Path) -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_journal_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
"workflow_journal_test", ROOT / "s16_workflow_runtime" / "code.py"
|
||||
)
|
||||
run_id = "wf_corrupt_0001"
|
||||
(tmp_path / f"{run_id}.journal.jsonl").write_text("{not-json}\n")
|
||||
@@ -133,7 +141,7 @@ def test_workflow_tool_adapter_uses_registry_and_returns_json(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_adapter_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
"workflow_adapter_test", ROOT / "s16_workflow_runtime" / "code.py"
|
||||
)
|
||||
monkeypatch.setattr(workflow, "STORE", tmp_path)
|
||||
|
||||
@@ -147,7 +155,7 @@ def test_workflow_tool_adapter_uses_registry_and_returns_json(
|
||||
assert result["launched"]["workflowName"] == "review-changes"
|
||||
assert result["task"]["status"] == "completed"
|
||||
assert result["task"]["taskType"] == "local_workflow"
|
||||
assert len(result["result"]["confirmed"]) == 6
|
||||
assert len(result["result"]["confirmed"]) == 5
|
||||
snapshot = json.loads(
|
||||
(tmp_path / f"{result['task']['runId']}.json").read_text()
|
||||
)
|
||||
@@ -162,7 +170,7 @@ def test_fresh_workflow_runs_have_unique_identity_and_resume_validates_args(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_identity_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
"workflow_identity_test", ROOT / "s16_workflow_runtime" / "code.py"
|
||||
)
|
||||
monkeypatch.setattr(workflow, "STORE", tmp_path)
|
||||
|
||||
@@ -185,7 +193,7 @@ def test_fresh_workflow_run_refuses_an_existing_identity(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_collision_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
"workflow_collision_test", ROOT / "s16_workflow_runtime" / "code.py"
|
||||
)
|
||||
monkeypatch.setattr(workflow, "STORE", tmp_path)
|
||||
fixed_id = "wf_review-changes_0000000000001a7b"
|
||||
@@ -207,7 +215,7 @@ def test_invalid_resume_does_not_overwrite_completed_artifacts(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_resume_guard_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
"workflow_resume_guard_test", ROOT / "s16_workflow_runtime" / "code.py"
|
||||
)
|
||||
monkeypatch.setattr(workflow, "STORE", tmp_path)
|
||||
result = asyncio.run(
|
||||
@@ -236,7 +244,7 @@ def test_active_workflow_run_rejects_concurrent_resume(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_active_run_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
"workflow_active_run_test", ROOT / "s16_workflow_runtime" / "code.py"
|
||||
)
|
||||
monkeypatch.setattr(workflow, "STORE", tmp_path)
|
||||
run_id = "wf_slow-test_0000000000001a7b"
|
||||
@@ -274,7 +282,7 @@ def test_active_workflow_run_rejects_concurrent_resume(
|
||||
|
||||
def test_workflow_run_lock_is_cross_process(tmp_path: Path) -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_process_lock_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
"workflow_process_lock_test", ROOT / "s16_workflow_runtime" / "code.py"
|
||||
)
|
||||
workflow.STORE = tmp_path
|
||||
run_id = "wf_process-lock_0000000000001a7b"
|
||||
@@ -284,7 +292,7 @@ def test_workflow_run_lock_is_cross_process(tmp_path: Path) -> None:
|
||||
with workflow.workflow_run_lock(run_id):
|
||||
child = context.Process(
|
||||
target=acquire_workflow_lock_in_child,
|
||||
args=(str(ROOT / "s18_workflow_runtime" / "code.py"),
|
||||
args=(str(ROOT / "s16_workflow_runtime" / "code.py"),
|
||||
str(tmp_path), run_id, results),
|
||||
)
|
||||
child.start()
|
||||
@@ -296,7 +304,7 @@ def test_workflow_run_lock_is_cross_process(tmp_path: Path) -> None:
|
||||
|
||||
def test_workflow_tool_extends_the_integrated_host_pool() -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_host_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
"workflow_host_test", ROOT / "s16_workflow_runtime" / "code.py"
|
||||
)
|
||||
host = types.SimpleNamespace(
|
||||
assemble_tool_pool=lambda: (
|
||||
@@ -312,13 +320,152 @@ def test_workflow_tool_extends_the_integrated_host_pool() -> None:
|
||||
assert handlers["Workflow"] is workflow.run_workflow_sync
|
||||
|
||||
|
||||
def test_workflow_default_entry_extends_the_real_s17_host(
|
||||
def test_anthropic_runner_parses_json_and_records_real_usage() -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_real_runner_test", ROOT / "s16_workflow_runtime" / "code.py"
|
||||
)
|
||||
calls = []
|
||||
|
||||
def create(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return types.SimpleNamespace(
|
||||
content=[types.SimpleNamespace(
|
||||
type="text", text='```json\n{"ok": true}\n```'
|
||||
)],
|
||||
usage=types.SimpleNamespace(input_tokens=11, output_tokens=7),
|
||||
)
|
||||
|
||||
client = types.SimpleNamespace(
|
||||
messages=types.SimpleNamespace(create=create)
|
||||
)
|
||||
runner = workflow.AnthropicAgentRunner(client, "deepseek-v4-flash")
|
||||
|
||||
result = runner.run(
|
||||
"Check the supplied change.",
|
||||
schema={
|
||||
"type": "object",
|
||||
"required": ["ok"],
|
||||
"properties": {"ok": {"type": "boolean"}},
|
||||
},
|
||||
label="check",
|
||||
)
|
||||
|
||||
assert result.value == {"ok": True}
|
||||
assert result.tokens == 18
|
||||
assert calls[0]["model"] == "deepseek-v4-flash"
|
||||
assert "tools" not in calls[0]
|
||||
|
||||
|
||||
def test_real_runner_output_retries_once_after_invalid_json(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_real_runner_retry_test",
|
||||
ROOT / "s16_workflow_runtime" / "code.py",
|
||||
)
|
||||
responses = iter([
|
||||
types.SimpleNamespace(
|
||||
content=[types.SimpleNamespace(type="text", text="not json")],
|
||||
usage=types.SimpleNamespace(input_tokens=3, output_tokens=2),
|
||||
),
|
||||
types.SimpleNamespace(
|
||||
content=[types.SimpleNamespace(
|
||||
type="text", text='Result:\n```json\n{"ok": true}\n```\nDone.'
|
||||
)],
|
||||
usage=types.SimpleNamespace(input_tokens=4, output_tokens=3),
|
||||
),
|
||||
])
|
||||
client = types.SimpleNamespace(
|
||||
messages=types.SimpleNamespace(create=lambda **_kwargs: next(responses))
|
||||
)
|
||||
runner = workflow.AnthropicAgentRunner(client, "test-model")
|
||||
journal = workflow.WorkflowJournal(
|
||||
"wf_json-retry_0001", resume=False, store=tmp_path
|
||||
)
|
||||
task = workflow.LocalWorkflowTask("task", "wf_json-retry_0001", {})
|
||||
state = workflow.ExecutionState(
|
||||
task, journal, runner, workflow.Budget(), {}
|
||||
)
|
||||
|
||||
try:
|
||||
result = asyncio.run(state.agent(
|
||||
"Return a result.",
|
||||
schema={
|
||||
"type": "object",
|
||||
"required": ["ok"],
|
||||
"properties": {"ok": {"type": "boolean"}},
|
||||
},
|
||||
label="json-retry",
|
||||
))
|
||||
finally:
|
||||
journal.close()
|
||||
|
||||
assert result == {"ok": True}
|
||||
assert task.usage == {"agents": 1, "tokens": 12}
|
||||
|
||||
|
||||
def test_install_workflow_tool_selects_the_host_api_runner() -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_runner_factory_test",
|
||||
ROOT / "s16_workflow_runtime" / "code.py",
|
||||
)
|
||||
client = object()
|
||||
host = types.SimpleNamespace(
|
||||
client=client,
|
||||
MODEL="deepseek-v4-flash",
|
||||
assemble_tool_pool=lambda: ([], {}),
|
||||
)
|
||||
|
||||
workflow.install_workflow_tool(host)
|
||||
runner = workflow.RUNNER_FACTORY()
|
||||
|
||||
assert isinstance(runner, workflow.AnthropicAgentRunner)
|
||||
assert runner.client is client
|
||||
assert runner.model == "deepseek-v4-flash"
|
||||
|
||||
|
||||
def test_parallel_agent_calls_do_not_block_the_event_loop(tmp_path: Path) -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_parallel_runner_test",
|
||||
ROOT / "s16_workflow_runtime" / "code.py",
|
||||
)
|
||||
barrier = threading.Barrier(2)
|
||||
|
||||
class BarrierRunner:
|
||||
def run(self, prompt, schema=None, label=None):
|
||||
barrier.wait(timeout=2)
|
||||
return workflow.RunnerOutput({"label": label}, 1)
|
||||
|
||||
journal = workflow.WorkflowJournal(
|
||||
"wf_parallel-test_0001", resume=False, store=tmp_path
|
||||
)
|
||||
task = workflow.LocalWorkflowTask("task", "wf_parallel-test_0001", {})
|
||||
state = workflow.ExecutionState(
|
||||
task, journal, BarrierRunner(), workflow.Budget(), {}
|
||||
)
|
||||
|
||||
async def run():
|
||||
return await state.parallel([
|
||||
lambda: state.agent("first", label="first"),
|
||||
lambda: state.agent("second", label="second"),
|
||||
])
|
||||
|
||||
try:
|
||||
result = asyncio.run(run())
|
||||
finally:
|
||||
journal.close()
|
||||
|
||||
assert result == [{"label": "first"}, {"label": "second"}]
|
||||
assert task.usage == {"agents": 2, "tokens": 2}
|
||||
|
||||
|
||||
def test_workflow_default_entry_extends_the_real_s15_host(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setenv("MODEL_ID", "test-model")
|
||||
workflow = load_lesson(
|
||||
"workflow_real_host_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
"workflow_real_host_test", ROOT / "s16_workflow_runtime" / "code.py"
|
||||
)
|
||||
host = workflow.load_integrated_host()
|
||||
|
||||
@@ -326,7 +473,7 @@ def test_workflow_default_entry_extends_the_real_s17_host(
|
||||
tools, handlers = host.assemble_tool_pool()
|
||||
names = [tool["name"] for tool in tools]
|
||||
|
||||
assert len(host.BUILTIN_TOOLS) == 24
|
||||
assert len(host.BUILTIN_TOOLS) == 25
|
||||
assert names[:-1] == [tool["name"] for tool in host.BUILTIN_TOOLS]
|
||||
assert names[-1] == "Workflow"
|
||||
assert handlers["Workflow"] is workflow.run_workflow_sync
|
||||
@@ -337,7 +484,7 @@ def test_workflow_default_entry_extends_the_real_s17_host(
|
||||
|
||||
def test_workflow_tool_adapter_rejects_model_supplied_code() -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_schema_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
"workflow_schema_test", ROOT / "s16_workflow_runtime" / "code.py"
|
||||
)
|
||||
properties = workflow.WORKFLOW_TOOL["input_schema"]["properties"]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user