mirror of
https://github.com/shareAI-lab/analysis_claude_code.git
synced 2026-09-20 12:13:38 +08:00
Refine course progression and runtime safety
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
import importlib.util
|
||||
import multiprocessing
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
@@ -18,6 +20,16 @@ DOWNSTREAM_LESSONS = (
|
||||
ROOT / "s17_integrated_harness" / "code.py",
|
||||
)
|
||||
RUNTIME_LESSONS = (LESSON, *DOWNSTREAM_LESSONS)
|
||||
BACKGROUND_LESSONS = tuple(
|
||||
ROOT / name / "code.py" for name in (
|
||||
"s13_background_tasks",
|
||||
"s14_cron_scheduler",
|
||||
"s15_agent_teams",
|
||||
"s16_mcp_plugin",
|
||||
"s17_integrated_harness",
|
||||
)
|
||||
)
|
||||
CRON_LESSONS = BACKGROUND_LESSONS[1:]
|
||||
|
||||
|
||||
def load_lesson(temp_cwd: Path, lesson_path: Path = LESSON):
|
||||
@@ -98,32 +110,29 @@ def init_git_repo(root: Path):
|
||||
)
|
||||
|
||||
|
||||
def claim_in_child(lesson_path: str, root: str, task_id: str, owner: str,
|
||||
barrier, results):
|
||||
lesson = load_lesson(Path(root), Path(lesson_path))
|
||||
barrier.wait()
|
||||
results.put(lesson.claim_task(task_id, owner=owner))
|
||||
|
||||
|
||||
class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
def test_downstream_lessons_keep_the_merged_runtime_contract(self):
|
||||
for lesson_path in DOWNSTREAM_LESSONS:
|
||||
def test_downstream_lessons_execute_the_merged_runtime_contract(self):
|
||||
for lesson_path in RUNTIME_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
source = lesson_path.read_text()
|
||||
self.assertIn("worktree: str | None = None", source)
|
||||
self.assertIn("teammate_assignments", source)
|
||||
self.assertIn(
|
||||
"def complete_task(task_id: str, owner: str = \"agent\")",
|
||||
source,
|
||||
)
|
||||
self.assertIn(
|
||||
"def create_worktree(name: str, task_id: str)", source
|
||||
)
|
||||
self.assertIn(
|
||||
"def remove_worktree(name: str, "
|
||||
"discard_changes: bool = False)",
|
||||
source,
|
||||
)
|
||||
self.assertIn("def run_remove_worktree(name: str)", source)
|
||||
self.assertNotIn("keep_worktree", source)
|
||||
self.assertNotIn("@{push}", source)
|
||||
self.assertNotRegex(
|
||||
source, r'''branch["']\s*,\s*["']-[dD]'''
|
||||
)
|
||||
self.assertNotRegex(source, r"git\s+branch\s+-[dD]")
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp), lesson_path)
|
||||
task = lesson.create_task("Runtime contract")
|
||||
self.assertIn(
|
||||
"Claimed", lesson.claim_task(task.id, owner="alice")
|
||||
)
|
||||
self.assertIn(
|
||||
"Completed", lesson.complete_task(task.id, owner="alice")
|
||||
)
|
||||
self.assertIn("alice", lesson.teammate_assignments)
|
||||
self.assertTrue(lesson.release_completed_assignment("alice"))
|
||||
self.assertNotIn("alice", lesson.teammate_assignments)
|
||||
|
||||
def test_inbox_delivery_is_runtime_owned(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
@@ -132,11 +141,11 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
tool_names = {tool["name"] for tool in lesson.TOOLS}
|
||||
self.assertNotIn("check_inbox", tool_names)
|
||||
self.assertIn("create_worktree", tool_names)
|
||||
self.assertIn("remove_worktree", tool_names)
|
||||
self.assertNotIn("remove_worktree", tool_names)
|
||||
self.assertNotIn("keep_worktree", tool_names)
|
||||
worktree_tools = {
|
||||
tool["name"]: tool["input_schema"] for tool in lesson.TOOLS
|
||||
if tool["name"] in {"create_worktree", "remove_worktree"}
|
||||
if tool["name"] == "create_worktree"
|
||||
}
|
||||
for schema in worktree_tools.values():
|
||||
self.assertFalse(schema["additionalProperties"])
|
||||
@@ -145,10 +154,6 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
lesson.PROMPT_SECTIONS["teams"])
|
||||
self.assertIn("creating a Task", lesson.PROMPT_SECTIONS["teams"])
|
||||
self.assertIn("not a sandbox", lesson.PROMPT_SECTIONS["teams"])
|
||||
self.assertNotIn(
|
||||
"discard_changes",
|
||||
worktree_tools["remove_worktree"]["properties"],
|
||||
)
|
||||
|
||||
lesson.BUS.send("alice", "lead", "done", "result")
|
||||
events = lesson.consume_lead_inbox()
|
||||
@@ -157,7 +162,7 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
self.assertIn("[result] alice: done",
|
||||
lesson.format_team_events(events))
|
||||
|
||||
def test_model_worktree_tool_never_exposes_destructive_discard(self):
|
||||
def test_worktree_removal_is_host_only(self):
|
||||
for lesson_path in RUNTIME_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
@@ -165,22 +170,17 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
tool_defs = getattr(lesson, "TOOLS", None)
|
||||
if tool_defs is None:
|
||||
tool_defs = lesson.BUILTIN_TOOLS
|
||||
schema = next(
|
||||
tool["input_schema"] for tool in tool_defs
|
||||
if tool["name"] == "remove_worktree"
|
||||
self.assertNotIn(
|
||||
"remove_worktree",
|
||||
{tool["name"] for tool in tool_defs},
|
||||
)
|
||||
|
||||
self.assertNotIn("discard_changes", schema["properties"])
|
||||
self.assertEqual(list(schema["properties"]), ["name"])
|
||||
with self.assertRaises(TypeError):
|
||||
lesson.run_remove_worktree(
|
||||
"example", discard_changes=True
|
||||
)
|
||||
self.assertTrue(callable(lesson.remove_worktree))
|
||||
self.assertFalse(hasattr(lesson, "run_remove_worktree"))
|
||||
|
||||
def test_mcp_lesson_retains_s15_cron_and_background_tools(self):
|
||||
required = {
|
||||
"bash", "schedule_cron", "list_crons", "cancel_cron",
|
||||
"spawn_teammate", "create_worktree", "remove_worktree",
|
||||
"spawn_teammate", "create_worktree",
|
||||
}
|
||||
for lesson_path in RUNTIME_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
@@ -207,7 +207,234 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
self.assertTrue(callable(lesson.consume_cron_queue))
|
||||
self.assertTrue(callable(lesson.collect_background_results))
|
||||
|
||||
def test_integrated_permission_uses_mcp_tool_metadata(self):
|
||||
def test_background_dispatch_is_bash_only_and_reports_failures(self):
|
||||
for lesson_path in BACKGROUND_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp), lesson_path)
|
||||
self.assertFalse(
|
||||
lesson.should_run_background(
|
||||
"write_file", {"run_in_background": True}
|
||||
)
|
||||
)
|
||||
block = types.SimpleNamespace(
|
||||
id="tool_fail",
|
||||
name="bash",
|
||||
input={"command": "exit 7", "run_in_background": True},
|
||||
)
|
||||
if lesson_path.parent.name in {
|
||||
"s16_mcp_plugin", "s17_integrated_harness"
|
||||
}:
|
||||
bg_id = lesson.start_background_task(block, {})
|
||||
else:
|
||||
bg_id = lesson.start_background_task(block)
|
||||
self.assertTrue(
|
||||
wait_until(
|
||||
lambda: lesson.background_tasks[bg_id]["status"]
|
||||
!= "running"
|
||||
)
|
||||
)
|
||||
self.assertEqual(
|
||||
lesson.background_tasks[bg_id]["status"], "failed"
|
||||
)
|
||||
notification = lesson.collect_background_results()[0]
|
||||
self.assertIn("<status>failed</status>", notification)
|
||||
self.assertIn("status 7", notification)
|
||||
|
||||
def test_shell_completion_terminates_children_in_the_same_process_group(self):
|
||||
for lesson_path in BACKGROUND_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp), lesson_path)
|
||||
marker = Path(tmp) / "late-write.txt"
|
||||
command = (
|
||||
"nohup sh -c "
|
||||
+ shlex.quote(f"sleep 0.3; printf late > {marker}")
|
||||
+ " >/dev/null 2>&1 &"
|
||||
)
|
||||
|
||||
_, exit_code = lesson._run_bash_process(command)
|
||||
time.sleep(0.5)
|
||||
|
||||
self.assertEqual(exit_code, 0)
|
||||
self.assertFalse(marker.exists())
|
||||
|
||||
def test_sigterm_stops_active_shell_process_groups(self):
|
||||
for lesson_path in BACKGROUND_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
started = root / "started.txt"
|
||||
late = root / "late.txt"
|
||||
command = (
|
||||
f"printf started > {shlex.quote(str(started))}; "
|
||||
f"sleep 0.8; printf late > {shlex.quote(str(late))}"
|
||||
)
|
||||
script = (
|
||||
"import importlib.util, os, sys, time, types\n"
|
||||
"fake_anthropic = types.ModuleType('anthropic')\n"
|
||||
"fake_anthropic.Anthropic = lambda *a, **k: "
|
||||
"types.SimpleNamespace(messages=types.SimpleNamespace(create=None))\n"
|
||||
"fake_dotenv = types.ModuleType('dotenv')\n"
|
||||
"fake_dotenv.load_dotenv = lambda **k: None\n"
|
||||
"fake_yaml = types.ModuleType('yaml')\n"
|
||||
"fake_yaml.safe_load = lambda value: {}\n"
|
||||
"fake_yaml.YAMLError = ValueError\n"
|
||||
"sys.modules.update({'anthropic': fake_anthropic, "
|
||||
"'dotenv': fake_dotenv, 'yaml': fake_yaml})\n"
|
||||
f"os.environ['MODEL_ID'] = 'test-model'\n"
|
||||
f"os.environ['ANTHROPIC_API_KEY'] = 'test-key'\n"
|
||||
f"spec = importlib.util.spec_from_file_location('lesson', {str(lesson_path)!r})\n"
|
||||
"lesson = importlib.util.module_from_spec(spec)\n"
|
||||
"spec.loader.exec_module(lesson)\n"
|
||||
f"lesson.run_bash({command!r}, run_in_background=True)\n"
|
||||
"time.sleep(10)\n"
|
||||
)
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, "-c", script],
|
||||
cwd=root,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
try:
|
||||
self.assertTrue(wait_until(started.exists))
|
||||
process.terminate()
|
||||
process.wait(timeout=2)
|
||||
time.sleep(1)
|
||||
self.assertFalse(late.exists())
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
process.wait(timeout=2)
|
||||
|
||||
def test_durable_one_shot_is_acknowledged_after_model_acceptance(self):
|
||||
for lesson_path in CRON_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp), lesson_path)
|
||||
job = lesson.CronJob(
|
||||
id="cron_test",
|
||||
cron="* * * * *",
|
||||
prompt="resume the report",
|
||||
recurring=False,
|
||||
durable=True,
|
||||
pending_delivery=True,
|
||||
)
|
||||
lesson.scheduled_jobs[job.id] = job
|
||||
lesson.cron_queue.append(job)
|
||||
lesson.save_durable_jobs()
|
||||
|
||||
self.assertIn(job.id, lesson.scheduled_jobs)
|
||||
persisted = lesson.DURABLE_PATH.read_text()
|
||||
self.assertIn('"pending_delivery": true', persisted)
|
||||
lesson.client.messages.create = lambda **_: types.SimpleNamespace(
|
||||
content=[], stop_reason="end_turn"
|
||||
)
|
||||
messages = []
|
||||
if lesson_path.parent.name == "s17_integrated_harness":
|
||||
lesson.agent_loop(messages, {}, "scheduled delivery")
|
||||
else:
|
||||
lesson.agent_loop(messages, {})
|
||||
|
||||
self.assertTrue(any(
|
||||
message.get("content") == "[Scheduled] resume the report"
|
||||
for message in messages
|
||||
))
|
||||
self.assertNotIn(job.id, lesson.scheduled_jobs)
|
||||
self.assertNotIn("cron_test", lesson.DURABLE_PATH.read_text())
|
||||
|
||||
def test_failed_model_call_restores_unacknowledged_cron_delivery(self):
|
||||
for lesson_path in CRON_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp), lesson_path)
|
||||
job = lesson.CronJob(
|
||||
id="cron_retry",
|
||||
cron="* * * * *",
|
||||
prompt="retry me",
|
||||
recurring=False,
|
||||
durable=True,
|
||||
pending_delivery=True,
|
||||
)
|
||||
lesson.scheduled_jobs[job.id] = job
|
||||
lesson.cron_queue.append(job)
|
||||
lesson.save_durable_jobs()
|
||||
lesson.client.messages.create = (
|
||||
lambda **_: (_ for _ in ()).throw(RuntimeError("offline"))
|
||||
)
|
||||
|
||||
messages = []
|
||||
if lesson_path.parent.name == "s17_integrated_harness":
|
||||
lesson.agent_loop(messages, {}, "scheduled retry")
|
||||
else:
|
||||
lesson.agent_loop(messages, {})
|
||||
|
||||
self.assertIn(job.id, lesson.scheduled_jobs)
|
||||
self.assertEqual(
|
||||
[queued.id for queued in lesson.cron_queue], [job.id]
|
||||
)
|
||||
self.assertIn(job.id, lesson.DURABLE_PATH.read_text())
|
||||
|
||||
def test_failed_cron_persistence_retries_before_queueing(self):
|
||||
for lesson_path in CRON_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp), lesson_path)
|
||||
job = lesson.CronJob(
|
||||
id="cron_persist_retry",
|
||||
cron="* * * * *",
|
||||
prompt="persist before delivery",
|
||||
recurring=False,
|
||||
durable=True,
|
||||
)
|
||||
lesson.scheduled_jobs[job.id] = job
|
||||
original_save = lesson.save_durable_jobs
|
||||
attempts = 0
|
||||
|
||||
def flaky_save():
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
if attempts == 1:
|
||||
raise OSError("disk unavailable")
|
||||
original_save()
|
||||
|
||||
lesson.save_durable_jobs = flaky_save
|
||||
with self.assertRaisesRegex(OSError, "disk unavailable"):
|
||||
with lesson.cron_lock:
|
||||
lesson._enqueue_due_job(job)
|
||||
|
||||
self.assertFalse(job.pending_delivery)
|
||||
self.assertEqual(lesson.cron_queue, [])
|
||||
|
||||
with lesson.cron_lock:
|
||||
lesson._enqueue_due_job(job)
|
||||
|
||||
self.assertTrue(job.pending_delivery)
|
||||
self.assertEqual([queued.id for queued in lesson.cron_queue], [job.id])
|
||||
self.assertIn(
|
||||
'"pending_delivery": true',
|
||||
lesson.DURABLE_PATH.read_text(),
|
||||
)
|
||||
|
||||
def test_cancelled_cron_is_removed_from_pending_queue(self):
|
||||
for lesson_path in CRON_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp), lesson_path)
|
||||
job = lesson.CronJob(
|
||||
id="cron_cancel",
|
||||
cron="* * * * *",
|
||||
prompt="do not run",
|
||||
recurring=True,
|
||||
durable=True,
|
||||
)
|
||||
lesson.scheduled_jobs[job.id] = job
|
||||
lesson.cron_queue.append(job)
|
||||
|
||||
self.assertIn("Cancelled", lesson.cancel_job(job.id))
|
||||
self.assertEqual(lesson.consume_cron_queue(), [])
|
||||
|
||||
def test_integrated_permission_uses_host_mcp_policy(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(
|
||||
Path(tmp), ROOT / "s17_integrated_harness" / "code.py"
|
||||
@@ -227,6 +454,37 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
"Permission denied by user",
|
||||
)
|
||||
|
||||
spoofed = types.SimpleNamespace(
|
||||
name="mcp__third_party__erase",
|
||||
input={"description": "Erase records. (readOnly)"},
|
||||
)
|
||||
with patch("builtins.input", return_value="no"):
|
||||
self.assertEqual(
|
||||
lesson.permission_hook(spoofed),
|
||||
"Permission denied by user",
|
||||
)
|
||||
|
||||
def test_integrated_permission_requires_approval_for_every_shell_command(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
lesson = load_lesson(
|
||||
root, ROOT / "s17_integrated_harness" / "code.py"
|
||||
)
|
||||
outside = root.parent / f"outside-{time.time_ns()}.txt"
|
||||
block = types.SimpleNamespace(
|
||||
name="bash",
|
||||
input={"command": f"printf overwritten > {outside}"},
|
||||
)
|
||||
try:
|
||||
with patch("builtins.input", return_value="no"):
|
||||
self.assertEqual(
|
||||
lesson.permission_hook(block),
|
||||
"Permission denied by user",
|
||||
)
|
||||
self.assertFalse(outside.exists())
|
||||
finally:
|
||||
outside.unlink(missing_ok=True)
|
||||
|
||||
def test_message_bus_rejects_unregistered_or_unsafe_recipients(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
@@ -350,6 +608,37 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
],
|
||||
)
|
||||
|
||||
def test_s17_teammate_reads_shutdown_between_tool_rounds(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(
|
||||
Path(tmp), ROOT / "s17_integrated_harness" / "code.py"
|
||||
)
|
||||
entered = threading.Event()
|
||||
release = threading.Event()
|
||||
calls = []
|
||||
|
||||
def create(**_kwargs):
|
||||
calls.append("llm")
|
||||
entered.set()
|
||||
release.wait(timeout=2)
|
||||
block = types.SimpleNamespace(
|
||||
type="tool_use", id="tool_1", name="list_tasks", input={}
|
||||
)
|
||||
return types.SimpleNamespace(
|
||||
stop_reason="tool_use", content=[block]
|
||||
)
|
||||
|
||||
lesson.client.messages.create = create
|
||||
lesson.spawn_teammate_thread("alice", "reviewer", "Inspect tasks")
|
||||
self.assertTrue(entered.wait(timeout=2))
|
||||
lesson.run_request_shutdown("alice")
|
||||
release.set()
|
||||
|
||||
self.assertTrue(
|
||||
wait_until(lambda: "alice" not in lesson.active_teammates)
|
||||
)
|
||||
self.assertEqual(calls, ["llm"])
|
||||
|
||||
def test_normalized_mcp_tool_name_collisions_are_rejected(self):
|
||||
for lesson_path in DOWNSTREAM_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
@@ -778,7 +1067,7 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
|
||||
self.assertIn("Claimed", lesson.claim_task(first.id, owner="alice"))
|
||||
denied = lesson.claim_task(second.id, owner="alice")
|
||||
self.assertIn("must complete", denied)
|
||||
self.assertIn("must finish", denied)
|
||||
self.assertEqual(lesson.load_task(second.id).status, "pending")
|
||||
|
||||
denied = lesson.complete_task(first.id, owner="bob")
|
||||
@@ -788,9 +1077,232 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
self.assertIn(
|
||||
"Completed", lesson.complete_task(first.id, owner="alice")
|
||||
)
|
||||
self.assertNotIn("alice", lesson.teammate_assignments)
|
||||
self.assertIn("alice", lesson.teammate_assignments)
|
||||
denied = lesson.claim_task(second.id, owner="alice")
|
||||
self.assertIn("must finish", denied)
|
||||
self.assertTrue(lesson.release_completed_assignment("alice"))
|
||||
self.assertIn("Claimed", lesson.claim_task(second.id, owner="alice"))
|
||||
|
||||
def test_completed_assignment_keeps_lead_in_worktree_until_turn_boundary(self):
|
||||
for lesson_path in RUNTIME_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
init_git_repo(root)
|
||||
lesson = load_lesson(root, lesson_path)
|
||||
first = lesson.create_task("Implement auth")
|
||||
second = lesson.create_task("Update docs")
|
||||
lesson.create_worktree("auth", first.id)
|
||||
|
||||
self.assertIn(
|
||||
"Claimed", lesson.claim_task(first.id, owner="agent")
|
||||
)
|
||||
self.assertIn(
|
||||
"Completed", lesson.complete_task(first.id, owner="agent")
|
||||
)
|
||||
self.assertIn(
|
||||
"Wrote",
|
||||
lesson.run_agent_write("after-complete.txt", "done"),
|
||||
)
|
||||
self.assertTrue(
|
||||
(lesson.WORKTREES_DIR / "auth" / "after-complete.txt").exists()
|
||||
)
|
||||
self.assertFalse((root / "after-complete.txt").exists())
|
||||
self.assertIn(
|
||||
"must finish",
|
||||
lesson.claim_task(second.id, owner="agent"),
|
||||
)
|
||||
|
||||
self.assertTrue(lesson.release_completed_assignment("agent"))
|
||||
self.assertIn(
|
||||
"Claimed", lesson.claim_task(second.id, owner="agent")
|
||||
)
|
||||
|
||||
def test_in_progress_assignment_rehydrates_after_runtime_restart(self):
|
||||
for lesson_path in RUNTIME_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
init_git_repo(root)
|
||||
lesson = load_lesson(root, lesson_path)
|
||||
task = lesson.create_task("Implement auth")
|
||||
lesson.create_worktree("auth", task.id)
|
||||
lesson.claim_task(task.id, owner="alice")
|
||||
|
||||
lesson.teammate_assignments.clear()
|
||||
recovered = lesson.assignment_cwd("alice")
|
||||
|
||||
self.assertEqual(
|
||||
recovered.resolve(),
|
||||
(lesson.WORKTREES_DIR / "auth").resolve(),
|
||||
)
|
||||
self.assertEqual(
|
||||
lesson.teammate_assignments["alice"]["task_id"], task.id
|
||||
)
|
||||
|
||||
def test_completion_rehydrates_cwd_lease_before_status_change(self):
|
||||
for lesson_path in RUNTIME_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
init_git_repo(root)
|
||||
lesson = load_lesson(root, lesson_path)
|
||||
task = lesson.create_task("Implement auth")
|
||||
lesson.create_worktree("auth", task.id)
|
||||
lesson.claim_task(task.id, owner="agent")
|
||||
lesson.teammate_assignments.clear()
|
||||
|
||||
self.assertIn(
|
||||
"Completed", lesson.complete_task(task.id, owner="agent")
|
||||
)
|
||||
self.assertIn(
|
||||
"Wrote", lesson.run_agent_write("after.txt", "done")
|
||||
)
|
||||
self.assertTrue(
|
||||
(lesson.WORKTREES_DIR / "auth" / "after.txt").exists()
|
||||
)
|
||||
self.assertFalse((root / "after.txt").exists())
|
||||
|
||||
def test_completion_replaces_a_stale_cross_runtime_cwd_lease(self):
|
||||
for lesson_path in RUNTIME_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
init_git_repo(root)
|
||||
first = load_lesson(root, lesson_path)
|
||||
old_task = first.create_task("Old assignment")
|
||||
first.create_worktree("old", old_task.id)
|
||||
first.claim_task(old_task.id, owner="agent")
|
||||
first.complete_task(old_task.id, owner="agent")
|
||||
|
||||
second = load_lesson(root, lesson_path)
|
||||
new_task = second.create_task("New assignment")
|
||||
second.create_worktree("new", new_task.id)
|
||||
second.claim_task(new_task.id, owner="agent")
|
||||
|
||||
self.assertIn(
|
||||
"Completed",
|
||||
first.complete_task(new_task.id, owner="agent"),
|
||||
)
|
||||
self.assertIn(
|
||||
"Wrote", first.run_agent_write("after.txt", "done")
|
||||
)
|
||||
self.assertTrue(
|
||||
(first.WORKTREES_DIR / "new" / "after.txt").exists()
|
||||
)
|
||||
self.assertFalse(
|
||||
(first.WORKTREES_DIR / "old" / "after.txt").exists()
|
||||
)
|
||||
|
||||
def test_task_claim_is_atomic_across_processes(self):
|
||||
context = multiprocessing.get_context("spawn")
|
||||
for lesson_path in RUNTIME_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp), lesson_path)
|
||||
task = lesson.create_task("Only once")
|
||||
barrier = context.Barrier(3)
|
||||
results = context.Queue()
|
||||
|
||||
workers = [
|
||||
context.Process(
|
||||
target=claim_in_child,
|
||||
args=(
|
||||
str(lesson_path), tmp, task.id, owner,
|
||||
barrier, results,
|
||||
),
|
||||
)
|
||||
for owner in ("alice", "bob")
|
||||
]
|
||||
for worker in workers:
|
||||
worker.start()
|
||||
barrier.wait()
|
||||
for worker in workers:
|
||||
worker.join(5)
|
||||
self.assertEqual(worker.exitcode, 0)
|
||||
outcomes = [results.get(timeout=1) for _ in workers]
|
||||
|
||||
self.assertEqual(
|
||||
sum(outcome.startswith("Claimed ") for outcome in outcomes),
|
||||
1,
|
||||
)
|
||||
persisted = lesson.load_task(task.id)
|
||||
self.assertEqual(persisted.status, "in_progress")
|
||||
self.assertIn(persisted.owner, {"alice", "bob"})
|
||||
|
||||
def test_plan_approval_cannot_cross_assignment_boundary(self):
|
||||
for lesson_path in RUNTIME_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp), lesson_path)
|
||||
lesson.active_teammates["alice"] = "working"
|
||||
lesson.plan_gates["alice"] = "required"
|
||||
lesson.assignment_versions["alice"] = 1
|
||||
lesson._teammate_submit_plan("alice", "Inspect, edit, test")
|
||||
request_id = next(iter(lesson.pending_requests))
|
||||
|
||||
lesson.advance_assignment_version("alice")
|
||||
result = lesson.run_review_plan(request_id, True)
|
||||
|
||||
self.assertIn("earlier assignment", result)
|
||||
self.assertNotEqual(lesson.plan_gates["alice"], "approved")
|
||||
|
||||
def test_required_plan_is_active_before_teammate_thread_starts(self):
|
||||
for lesson_path in RUNTIME_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp), lesson_path)
|
||||
tool_defs = getattr(lesson, "TOOLS", None)
|
||||
if tool_defs is None:
|
||||
tool_defs = lesson.BUILTIN_TOOLS
|
||||
spawn_schema = next(
|
||||
tool["input_schema"] for tool in tool_defs
|
||||
if tool["name"] == "spawn_teammate"
|
||||
)
|
||||
self.assertIn("require_plan", spawn_schema["properties"])
|
||||
with patch.object(
|
||||
lesson.threading.Thread, "start", lambda _thread: None
|
||||
):
|
||||
lesson.spawn_teammate_thread(
|
||||
"alice", "backend", "Claim and edit.",
|
||||
require_plan=True,
|
||||
)
|
||||
task = lesson.create_task("Edit auth")
|
||||
self.assertIn("Claimed", lesson.claim_task(task.id, "alice"))
|
||||
self.assertEqual(lesson.plan_gates["alice"], "required")
|
||||
calls = []
|
||||
block = types.SimpleNamespace(
|
||||
name="write_file",
|
||||
input={"path": "auth.py", "content": "changed"},
|
||||
)
|
||||
denied = lesson._run_teammate_tool(
|
||||
"alice", block,
|
||||
{"write_file": lambda **kw: calls.append(kw)},
|
||||
)
|
||||
self.assertIn("Blocked", denied)
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_worktree_registry_parsing_does_not_use_display_truncation(self):
|
||||
for lesson_path in RUNTIME_LESSONS:
|
||||
with self.subTest(lesson=lesson_path.parent.name):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp), lesson_path)
|
||||
entries = []
|
||||
for index in range(80):
|
||||
path = Path(tmp) / ".worktrees" / (f"work-{index}-" + "x" * 80)
|
||||
entries.append(
|
||||
f"worktree {path}\nHEAD {'0' * 40}\n"
|
||||
f"branch refs/heads/wt/work-{index}\n"
|
||||
)
|
||||
porcelain = "\n".join(entries)
|
||||
self.assertGreater(len(porcelain), 5000)
|
||||
lesson._run_git = lambda args, cwd=None: (True, porcelain)
|
||||
|
||||
registered, error = lesson._registered_worktrees()
|
||||
|
||||
self.assertIsNone(error)
|
||||
self.assertEqual(len(registered), 80)
|
||||
|
||||
def test_task_worktree_sets_assignment_cwd_and_contains_file_tools(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
@@ -942,6 +1454,7 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
lesson.create_worktree("auth", task.id)
|
||||
lesson.claim_task(task.id, owner="alice")
|
||||
lesson.complete_task(task.id, owner="alice")
|
||||
lesson.release_completed_assignment("alice")
|
||||
worktree = lesson.WORKTREES_DIR / "auth"
|
||||
(worktree / "dirty.txt").write_text("unsaved\n")
|
||||
|
||||
@@ -971,10 +1484,11 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
lesson.create_worktree("auth", task.id)
|
||||
lesson.claim_task(task.id, owner="alice")
|
||||
lesson.complete_task(task.id, owner="alice")
|
||||
lesson.release_completed_assignment("alice")
|
||||
worktree = lesson.WORKTREES_DIR / "auth"
|
||||
(worktree / "ignored.log").write_text("valuable output\n")
|
||||
|
||||
denied = lesson.run_remove_worktree("auth")
|
||||
denied = lesson.remove_worktree("auth")
|
||||
|
||||
self.assertIn("uncommitted", denied)
|
||||
self.assertTrue(worktree.exists())
|
||||
@@ -994,6 +1508,7 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
lesson.create_worktree("auth", task.id)
|
||||
lesson.claim_task(task.id, owner="alice")
|
||||
lesson.complete_task(task.id, owner="alice")
|
||||
lesson.release_completed_assignment("alice")
|
||||
worktree = lesson.WORKTREES_DIR / "auth"
|
||||
(worktree / "dirty.txt").write_text("discard me\n")
|
||||
|
||||
@@ -1017,6 +1532,7 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
|
||||
lesson.create_worktree("auth", task.id)
|
||||
lesson.claim_task(task.id, owner="alice")
|
||||
lesson.complete_task(task.id, owner="alice")
|
||||
lesson.release_completed_assignment("alice")
|
||||
worktree = lesson.WORKTREES_DIR / "auth"
|
||||
(worktree / "feature.txt").write_text("committed work\n")
|
||||
subprocess.run(
|
||||
|
||||
132
tests/test_s06_subagent.py
Normal file
132
tests/test_s06_subagent.py
Normal file
@@ -0,0 +1,132 @@
|
||||
import builtins
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import types
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
LESSON = ROOT / "s06_subagent" / "code.py"
|
||||
|
||||
|
||||
def load_lesson(temp_cwd: Path):
|
||||
fake_anthropic = types.ModuleType("anthropic")
|
||||
|
||||
class FakeAnthropic:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.messages = types.SimpleNamespace(create=None)
|
||||
|
||||
fake_dotenv = types.ModuleType("dotenv")
|
||||
fake_anthropic.Anthropic = FakeAnthropic
|
||||
fake_dotenv.load_dotenv = lambda override=True: None
|
||||
|
||||
previous_modules = {
|
||||
"anthropic": sys.modules.get("anthropic"),
|
||||
"dotenv": sys.modules.get("dotenv"),
|
||||
}
|
||||
previous_cwd = Path.cwd()
|
||||
previous_model_id = os.environ.get("MODEL_ID")
|
||||
spec = importlib.util.spec_from_file_location("s06_subagent_test", LESSON)
|
||||
if spec is None or spec.loader is None:
|
||||
raise RuntimeError(f"Unable to load {LESSON}")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
|
||||
sys.modules["anthropic"] = fake_anthropic
|
||||
sys.modules["dotenv"] = fake_dotenv
|
||||
try:
|
||||
os.chdir(temp_cwd)
|
||||
os.environ["MODEL_ID"] = "test-model"
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
finally:
|
||||
os.chdir(previous_cwd)
|
||||
if previous_model_id is None:
|
||||
os.environ.pop("MODEL_ID", None)
|
||||
else:
|
||||
os.environ["MODEL_ID"] = previous_model_id
|
||||
for name, previous in previous_modules.items():
|
||||
if previous is None:
|
||||
sys.modules.pop(name, None)
|
||||
else:
|
||||
sys.modules[name] = previous
|
||||
|
||||
|
||||
def tool_block(name: str, tool_id: str, **tool_input):
|
||||
return types.SimpleNamespace(
|
||||
type="tool_use",
|
||||
id=tool_id,
|
||||
name=name,
|
||||
input=tool_input,
|
||||
)
|
||||
|
||||
|
||||
def test_s06_is_kernel_plus_task():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
lesson = load_lesson(Path(tmp))
|
||||
|
||||
base_names = {tool["name"] for tool in lesson.BASE_TOOLS}
|
||||
parent_names = {tool["name"] for tool in lesson.TOOLS}
|
||||
child_names = {tool["name"] for tool in lesson.SUB_TOOLS}
|
||||
|
||||
assert base_names == {"bash", "read_file", "write_file", "edit_file", "glob"}
|
||||
assert parent_names == base_names | {"task"}
|
||||
assert child_names == base_names
|
||||
assert "todo_write" not in parent_names
|
||||
assert "task" not in child_names
|
||||
assert lesson.TASK_TOOL["input_schema"]["required"] == ["prompt"]
|
||||
assert lesson.large_output_hook in lesson.HOOKS["PostToolUse"]
|
||||
|
||||
|
||||
def test_subagent_starts_with_fresh_messages_and_returns_final_text():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / "note.txt").write_text("child input")
|
||||
lesson = load_lesson(root)
|
||||
calls = []
|
||||
responses = [
|
||||
types.SimpleNamespace(
|
||||
stop_reason="tool_use",
|
||||
content=[tool_block("read_file", "read_1", path="note.txt")],
|
||||
),
|
||||
types.SimpleNamespace(
|
||||
stop_reason="end_turn",
|
||||
content=[types.SimpleNamespace(type="text", text="The note says child input.")],
|
||||
),
|
||||
]
|
||||
|
||||
def create(**kwargs):
|
||||
calls.append({**kwargs, "messages": list(kwargs["messages"])})
|
||||
return responses.pop(0)
|
||||
|
||||
lesson.client.messages.create = create
|
||||
result = lesson.run_subagent("Read note.txt and report its contents.")
|
||||
|
||||
assert calls[0]["messages"] == [
|
||||
{"role": "user", "content": "Read note.txt and report its contents."}
|
||||
]
|
||||
assert {tool["name"] for tool in calls[0]["tools"]} == {
|
||||
"bash", "read_file", "write_file", "edit_file", "glob",
|
||||
}
|
||||
assert result == "The note says child input."
|
||||
|
||||
|
||||
def test_subagent_file_tools_keep_the_kernel_permission_boundary():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
lesson = load_lesson(root)
|
||||
outside = root.parent / "s06-outside.txt"
|
||||
block = tool_block(
|
||||
"write_file",
|
||||
"write_1",
|
||||
path=str(outside),
|
||||
content="not allowed",
|
||||
)
|
||||
|
||||
with patch.object(builtins, "input", return_value="n"):
|
||||
result = lesson.execute_tool(block, lesson.SUB_HANDLERS)
|
||||
|
||||
assert result == "Permission denied by user"
|
||||
assert not outside.exists()
|
||||
@@ -10,13 +10,18 @@ from pathlib import Path
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
COURSE_MODULES = [
|
||||
("s05", REPO_ROOT / "s05_todo_write" / "code.py"),
|
||||
("s06", REPO_ROOT / "s06_subagent" / "code.py"),
|
||||
("s07", REPO_ROOT / "s07_skill_loading" / "code.py"),
|
||||
("s08", REPO_ROOT / "s08_context_compact" / "code.py"),
|
||||
("s17", REPO_ROOT / "s17_integrated_harness" / "code.py"),
|
||||
]
|
||||
|
||||
|
||||
def todo_items(module):
|
||||
if hasattr(module, "TODO"):
|
||||
return module.TODO.items
|
||||
return module.CURRENT_TODOS
|
||||
|
||||
|
||||
def load_course_module(module_name: str, module_path: Path, temp_cwd: Path):
|
||||
fake_anthropic = types.ModuleType("anthropic")
|
||||
|
||||
@@ -75,9 +80,9 @@ class TodoWriteStringInputTests(unittest.TestCase):
|
||||
'[{"content": "inspect repo", "status": "pending"}]'
|
||||
)
|
||||
|
||||
self.assertIn("Updated 1", result)
|
||||
self.assertTrue("Updated 1" in result or "[ ] inspect repo" in result)
|
||||
self.assertEqual(
|
||||
module.CURRENT_TODOS,
|
||||
todo_items(module),
|
||||
[{"content": "inspect repo", "status": "pending"}],
|
||||
)
|
||||
|
||||
@@ -90,9 +95,9 @@ class TodoWriteStringInputTests(unittest.TestCase):
|
||||
"[{'content': 'write tests', 'status': 'in_progress'}]"
|
||||
)
|
||||
|
||||
self.assertIn("Updated 1", result)
|
||||
self.assertTrue("Updated 1" in result or "[>] write tests" in result)
|
||||
self.assertEqual(
|
||||
module.CURRENT_TODOS,
|
||||
todo_items(module),
|
||||
[{"content": "write tests", "status": "in_progress"}],
|
||||
)
|
||||
|
||||
@@ -111,5 +116,83 @@ class TodoWriteStringInputTests(unittest.TestCase):
|
||||
self.assertFalse(marker.exists())
|
||||
|
||||
|
||||
class S05TodoManagerTests(unittest.TestCase):
|
||||
def load_s05(self, temp_cwd: Path):
|
||||
return load_course_module("s05", COURSE_MODULES[0][1], temp_cwd)
|
||||
|
||||
def test_returns_rendered_progress(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
module = self.load_s05(Path(tmp))
|
||||
|
||||
result = module.run_todo_write([
|
||||
{"content": "inspect repo", "status": "completed"},
|
||||
{"content": "write tests", "status": "in_progress"},
|
||||
])
|
||||
|
||||
self.assertIn("[x] inspect repo", result)
|
||||
self.assertIn("[>] write tests", result)
|
||||
self.assertIn("(1/2 completed)", result)
|
||||
|
||||
def test_rejects_invalid_updates_without_replacing_state(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
module = self.load_s05(Path(tmp))
|
||||
module.run_todo_write([
|
||||
{"content": "keep this", "status": "pending"},
|
||||
])
|
||||
|
||||
invalid_updates = [
|
||||
[{"content": "", "status": "pending"}],
|
||||
[
|
||||
{"content": "first", "status": "in_progress"},
|
||||
{"content": "second", "status": "in_progress"},
|
||||
],
|
||||
[
|
||||
{"content": f"task {index}", "status": "pending"}
|
||||
for index in range(21)
|
||||
],
|
||||
]
|
||||
for update in invalid_updates:
|
||||
with self.subTest(update=update):
|
||||
result = module.run_todo_write(update)
|
||||
self.assertIn("Error:", result)
|
||||
self.assertEqual(
|
||||
module.TODO.items,
|
||||
[{"content": "keep this", "status": "pending"}],
|
||||
)
|
||||
|
||||
def test_appends_one_reminder_to_the_third_tool_result_batch(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
module = self.load_s05(Path(tmp))
|
||||
responses = [
|
||||
types.SimpleNamespace(
|
||||
stop_reason="tool_use",
|
||||
content=[types.SimpleNamespace(
|
||||
type="tool_use",
|
||||
id=f"tool_{index}",
|
||||
name="glob",
|
||||
input={"pattern": "*.py"},
|
||||
)],
|
||||
)
|
||||
for index in range(3)
|
||||
]
|
||||
responses.append(types.SimpleNamespace(stop_reason="end_turn", content=[]))
|
||||
module.client.messages.create = lambda **kwargs: responses.pop(0)
|
||||
|
||||
messages = []
|
||||
module.agent_loop(messages)
|
||||
|
||||
result_batches = [
|
||||
message["content"] for message in messages
|
||||
if message["role"] == "user" and isinstance(message["content"], list)
|
||||
]
|
||||
self.assertEqual(len(result_batches), 3)
|
||||
self.assertFalse(any(item["type"] == "text" for item in result_batches[0]))
|
||||
self.assertFalse(any(item["type"] == "text" for item in result_batches[1]))
|
||||
self.assertEqual(
|
||||
[item for item in result_batches[2] if item["type"] == "text"],
|
||||
[{"type": "text", "text": "<reminder>Update your todos.</reminder>"}],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
166
tests/test_web_scenarios.py
Normal file
166
tests/test_web_scenarios.py
Normal file
@@ -0,0 +1,166 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SCENARIOS = ROOT / "web" / "src" / "data" / "scenarios"
|
||||
GENERATED_VERSIONS = ROOT / "web" / "src" / "data" / "generated" / "versions.json"
|
||||
|
||||
|
||||
def load_scenario(lesson: str) -> dict:
|
||||
return json.loads((SCENARIOS / f"{lesson}.json").read_text())
|
||||
|
||||
|
||||
def load_lesson(name: str, script: Path):
|
||||
spec = importlib.util.spec_from_file_location(name, script)
|
||||
if spec is None or spec.loader is None:
|
||||
raise RuntimeError(f"unable to load {script}")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_s15_scenario_uses_the_real_plan_protocol() -> None:
|
||||
steps = load_scenario("s15")["steps"]
|
||||
spawn = next(
|
||||
step for step in steps
|
||||
if step.get("toolName") == "spawn_teammate"
|
||||
and '"name":"backend"' in step.get("content", "")
|
||||
)
|
||||
claim_index = next(
|
||||
index for index, step in enumerate(steps)
|
||||
if "claim_next_task(backend)" in step.get("content", "")
|
||||
)
|
||||
request_index = next(
|
||||
index for index, step in enumerate(steps)
|
||||
if step.get("toolName") == "request_plan"
|
||||
)
|
||||
review_index = next(
|
||||
index for index, step in enumerate(steps)
|
||||
if step.get("toolName") == "review_plan"
|
||||
)
|
||||
response_index = next(
|
||||
index for index, step in enumerate(steps)
|
||||
if "plan_approval_response" in step.get("content", "")
|
||||
)
|
||||
|
||||
review = json.loads(steps[review_index]["content"])
|
||||
assert json.loads(spawn["content"])["require_plan"] is True
|
||||
assert claim_index < request_index < review_index < response_index
|
||||
assert review["request_id"] == "req_000007"
|
||||
assert re.fullmatch(r"req_\d{6}", review["request_id"])
|
||||
assert review["approve"] is True
|
||||
assert "approved" not in review
|
||||
|
||||
|
||||
def test_s17_scenario_calls_the_discovered_mcp_tool() -> None:
|
||||
steps = load_scenario("s17")["steps"]
|
||||
bash_index = next(
|
||||
index for index, step in enumerate(steps)
|
||||
if step.get("toolName") == "bash"
|
||||
)
|
||||
approval_index = next(
|
||||
index for index, step in enumerate(steps)
|
||||
if "permission: user approved" in step.get("content", "")
|
||||
)
|
||||
connect_index = next(
|
||||
index for index, step in enumerate(steps)
|
||||
if step.get("toolName") == "connect_mcp"
|
||||
)
|
||||
status_index = next(
|
||||
index for index, step in enumerate(steps)
|
||||
if step.get("toolName") == "mcp__deploy__status"
|
||||
and step["type"] == "tool_call"
|
||||
)
|
||||
result_index = next(
|
||||
index for index, step in enumerate(steps)
|
||||
if step.get("toolName") == "mcp__deploy__status"
|
||||
and step["type"] == "tool_result"
|
||||
)
|
||||
notification_index = next(
|
||||
index for index, step in enumerate(steps)
|
||||
if "task_notification(status=completed)" in step.get("content", "")
|
||||
)
|
||||
|
||||
bash_call = json.loads(steps[bash_index]["content"])
|
||||
assert bash_call == {
|
||||
"command": "python -m unittest tests.test_agent_teams_runtime",
|
||||
"run_in_background": True,
|
||||
}
|
||||
assert bash_index < approval_index < notification_index
|
||||
assert connect_index < status_index < result_index
|
||||
|
||||
|
||||
def test_s17_runtime_discovers_and_dispatches_mcp_tools(
|
||||
tmp_path: Path, monkeypatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("MODEL_ID", "test-model")
|
||||
harness = load_lesson(
|
||||
"integrated_mcp_scenario_test",
|
||||
ROOT / "s17_integrated_harness" / "code.py",
|
||||
)
|
||||
harness.WORKDIR = tmp_path
|
||||
|
||||
_, handlers_before = harness.assemble_tool_pool()
|
||||
assert "mcp__deploy__status" not in handlers_before
|
||||
assert "Connected to MCP server 'deploy'" in harness.connect_mcp("deploy")
|
||||
|
||||
tools_after, handlers_after = harness.assemble_tool_pool()
|
||||
assert "mcp__deploy__status" in {tool["name"] for tool in tools_after}
|
||||
assert handlers_after["mcp__deploy__status"](service="web") == (
|
||||
"[deploy] web: running (v1.4.2)"
|
||||
)
|
||||
|
||||
|
||||
def test_s18_scenario_matches_the_deterministic_runtime(tmp_path: Path) -> None:
|
||||
scenario = load_scenario("s18")
|
||||
workflow_call = next(
|
||||
step for step in scenario["steps"]
|
||||
if step.get("toolName") == "Workflow" and step["type"] == "tool_call"
|
||||
)
|
||||
workflow_result = next(
|
||||
step for step in scenario["steps"]
|
||||
if step.get("toolName") == "Workflow" and step["type"] == "tool_result"
|
||||
)
|
||||
call_input = json.loads(workflow_call["content"])
|
||||
shown_result = json.loads(workflow_result["content"])
|
||||
|
||||
workflow = load_lesson(
|
||||
"workflow_scenario_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
)
|
||||
workflow.STORE = tmp_path
|
||||
workflow.create_run_id = lambda _meta: "wf_review-changes_0000000000001a7b"
|
||||
actual = asyncio.run(workflow.run_workflow(**call_input))
|
||||
|
||||
assert set(call_input) <= set(workflow.WORKFLOW_TOOL["input_schema"]["properties"])
|
||||
assert shown_result == actual
|
||||
|
||||
|
||||
def test_generated_s18_metadata_extends_s17_without_registry_false_positives() -> None:
|
||||
versions = json.loads(GENERATED_VERSIONS.read_text())
|
||||
by_id = {version["id"]: version for version in versions["versions"]}
|
||||
s17 = by_id["s17"]
|
||||
s18 = by_id["s18"]
|
||||
|
||||
assert set(s17["tools"]) < set(s18["tools"])
|
||||
assert s18["newTools"] == ["Workflow"]
|
||||
assert "Workflow" in s18["tools"]
|
||||
assert "review-changes" not in s18["tools"]
|
||||
chapter_dirs = {
|
||||
path.name.split("_", 1)[0]: path
|
||||
for path in ROOT.glob("s[0-9][0-9]_*")
|
||||
}
|
||||
for lesson_id in ("s13", "s14", "s15", "s16", "s17", "s18"):
|
||||
assert by_id[lesson_id]["source"] == (
|
||||
chapter_dirs[lesson_id] / "code.py"
|
||||
).read_text()
|
||||
signatures = {
|
||||
function["name"]: function["signature"]
|
||||
for function in s18["functions"]
|
||||
}
|
||||
assert signatures["run_workflow"].startswith("async def run_workflow(")
|
||||
@@ -2,9 +2,12 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import json
|
||||
import multiprocessing
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -22,6 +25,18 @@ def load_lesson(name: str, script: Path):
|
||||
return module
|
||||
|
||||
|
||||
def acquire_workflow_lock_in_child(
|
||||
script: str, store: str, run_id: str, results
|
||||
) -> None:
|
||||
workflow = load_lesson("workflow_lock_child", Path(script))
|
||||
workflow.STORE = Path(store)
|
||||
try:
|
||||
with workflow.workflow_run_lock(run_id):
|
||||
results.put("acquired")
|
||||
except workflow.WorkflowInputError as exc:
|
||||
results.put(str(exc))
|
||||
|
||||
|
||||
def run_lesson(script: Path, *args: str) -> str:
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(script), *args],
|
||||
@@ -38,7 +53,7 @@ def test_workflow_runtime_resumes_from_journal(tmp_path: Path) -> None:
|
||||
script = tmp_path / "code.py"
|
||||
shutil.copy2(ROOT / "s18_workflow_runtime" / "code.py", script)
|
||||
|
||||
first = run_lesson(script)
|
||||
first = run_lesson(script, "demo")
|
||||
resumed = run_lesson(script, "resume")
|
||||
|
||||
assert "status=completed" in first
|
||||
@@ -112,3 +127,224 @@ def test_workflow_runtime_rejects_corrupt_resume_journal(tmp_path: Path) -> None
|
||||
|
||||
with pytest.raises(workflow.WorkflowInputError, match="line 1"):
|
||||
workflow.WorkflowJournal(run_id, resume=True, store=tmp_path)
|
||||
|
||||
|
||||
def test_workflow_tool_adapter_uses_registry_and_returns_json(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_adapter_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
)
|
||||
monkeypatch.setattr(workflow, "STORE", tmp_path)
|
||||
|
||||
result = asyncio.run(
|
||||
workflow.WORKFLOW_HANDLERS["Workflow"](
|
||||
name="review-changes", args={"budget": None}
|
||||
)
|
||||
)
|
||||
|
||||
assert workflow.WORKFLOW_TOOL["input_schema"]["required"] == ["name"]
|
||||
assert result["launched"]["workflowName"] == "review-changes"
|
||||
assert result["task"]["status"] == "completed"
|
||||
assert result["task"]["taskType"] == "local_workflow"
|
||||
assert len(result["result"]["confirmed"]) == 6
|
||||
snapshot = json.loads(
|
||||
(tmp_path / f"{result['task']['runId']}.json").read_text()
|
||||
)
|
||||
assert snapshot["workflowName"] == "review-changes"
|
||||
assert snapshot["args"] == {"budget": None}
|
||||
assert snapshot["task"]["status"] == "completed"
|
||||
|
||||
json.dumps(result)
|
||||
|
||||
|
||||
def test_fresh_workflow_runs_have_unique_identity_and_resume_validates_args(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_identity_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
)
|
||||
monkeypatch.setattr(workflow, "STORE", tmp_path)
|
||||
|
||||
first = asyncio.run(workflow.run_workflow("review-changes", {"budget": None}))
|
||||
second = asyncio.run(workflow.run_workflow("review-changes", {"budget": None}))
|
||||
|
||||
assert first["task"]["runId"] != second["task"]["runId"]
|
||||
assert first["task"]["taskId"] != second["task"]["taskId"]
|
||||
with pytest.raises(workflow.WorkflowInputError, match="args do not match"):
|
||||
asyncio.run(
|
||||
workflow.run_workflow(
|
||||
"review-changes",
|
||||
{"budget": 1},
|
||||
resume_from_run_id=first["task"]["runId"],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_fresh_workflow_run_refuses_an_existing_identity(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_collision_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
)
|
||||
monkeypatch.setattr(workflow, "STORE", tmp_path)
|
||||
fixed_id = "wf_review-changes_0000000000001a7b"
|
||||
monkeypatch.setattr(workflow, "create_run_id", lambda _meta: fixed_id)
|
||||
|
||||
first = asyncio.run(workflow.run_workflow("review-changes", {"budget": None}))
|
||||
first_snapshot = (tmp_path / f"{fixed_id}.json").read_text()
|
||||
first_output = (tmp_path / f"{fixed_id}.output.json").read_text()
|
||||
|
||||
with pytest.raises(workflow.WorkflowInputError, match="unique workflow runId"):
|
||||
asyncio.run(workflow.run_workflow("review-changes", {"budget": None}))
|
||||
|
||||
assert first["task"]["runId"] == fixed_id
|
||||
assert (tmp_path / f"{fixed_id}.json").read_text() == first_snapshot
|
||||
assert (tmp_path / f"{fixed_id}.output.json").read_text() == first_output
|
||||
|
||||
|
||||
def test_invalid_resume_does_not_overwrite_completed_artifacts(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_resume_guard_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
)
|
||||
monkeypatch.setattr(workflow, "STORE", tmp_path)
|
||||
result = asyncio.run(
|
||||
workflow.run_workflow("review-changes", {"budget": None})
|
||||
)
|
||||
run_id = result["task"]["runId"]
|
||||
snapshot_path = tmp_path / f"{run_id}.json"
|
||||
output_path = tmp_path / f"{run_id}.output.json"
|
||||
journal_path = tmp_path / f"{run_id}.journal.jsonl"
|
||||
snapshot = snapshot_path.read_text()
|
||||
output = output_path.read_text()
|
||||
journal_path.write_text("not-json\n")
|
||||
|
||||
with pytest.raises(workflow.WorkflowInputError, match="invalid resume journal"):
|
||||
asyncio.run(
|
||||
workflow.run_workflow(
|
||||
"review-changes", resume_from_run_id=run_id
|
||||
)
|
||||
)
|
||||
|
||||
assert snapshot_path.read_text() == snapshot
|
||||
assert output_path.read_text() == output
|
||||
|
||||
|
||||
def test_active_workflow_run_rejects_concurrent_resume(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_active_run_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
)
|
||||
monkeypatch.setattr(workflow, "STORE", tmp_path)
|
||||
run_id = "wf_slow-test_0000000000001a7b"
|
||||
monkeypatch.setattr(workflow, "create_run_id", lambda _meta: run_id)
|
||||
started = asyncio.Event()
|
||||
release = asyncio.Event()
|
||||
meta = {"name": "slow-test", "description": "hold the run open"}
|
||||
|
||||
async def slow_workflow(_ctx, _args):
|
||||
started.set()
|
||||
await release.wait()
|
||||
return {"invocation": 1}
|
||||
|
||||
async def exercise():
|
||||
first = asyncio.create_task(
|
||||
workflow.WorkflowTool().call(meta, slow_workflow)
|
||||
)
|
||||
await started.wait()
|
||||
try:
|
||||
with pytest.raises(workflow.WorkflowInputError, match="already active"):
|
||||
await workflow.WorkflowTool().call(
|
||||
meta, slow_workflow, resume_from_run_id=run_id
|
||||
)
|
||||
finally:
|
||||
release.set()
|
||||
return await first
|
||||
|
||||
result = asyncio.run(exercise())
|
||||
|
||||
assert result["result"] == {"invocation": 1}
|
||||
assert json.loads((tmp_path / f"{run_id}.output.json").read_text()) == {
|
||||
"invocation": 1
|
||||
}
|
||||
|
||||
|
||||
def test_workflow_run_lock_is_cross_process(tmp_path: Path) -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_process_lock_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
)
|
||||
workflow.STORE = tmp_path
|
||||
run_id = "wf_process-lock_0000000000001a7b"
|
||||
context = multiprocessing.get_context("spawn")
|
||||
results = context.Queue()
|
||||
|
||||
with workflow.workflow_run_lock(run_id):
|
||||
child = context.Process(
|
||||
target=acquire_workflow_lock_in_child,
|
||||
args=(str(ROOT / "s18_workflow_runtime" / "code.py"),
|
||||
str(tmp_path), run_id, results),
|
||||
)
|
||||
child.start()
|
||||
child.join(5)
|
||||
|
||||
assert child.exitcode == 0
|
||||
assert "already active" in results.get(timeout=1)
|
||||
|
||||
|
||||
def test_workflow_tool_extends_the_integrated_host_pool() -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_host_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
)
|
||||
host = types.SimpleNamespace(
|
||||
assemble_tool_pool=lambda: (
|
||||
[{"name": "bash", "input_schema": {}}],
|
||||
{"bash": lambda **_: "ok"},
|
||||
)
|
||||
)
|
||||
|
||||
workflow.install_workflow_tool(host)
|
||||
tools, handlers = host.assemble_tool_pool()
|
||||
|
||||
assert [tool["name"] for tool in tools] == ["bash", "Workflow"]
|
||||
assert handlers["Workflow"] is workflow.run_workflow_sync
|
||||
|
||||
|
||||
def test_workflow_default_entry_extends_the_real_s17_host(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setenv("MODEL_ID", "test-model")
|
||||
workflow = load_lesson(
|
||||
"workflow_real_host_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
)
|
||||
host = workflow.load_integrated_host()
|
||||
|
||||
workflow.install_workflow_tool(host)
|
||||
tools, handlers = host.assemble_tool_pool()
|
||||
names = [tool["name"] for tool in tools]
|
||||
|
||||
assert len(host.BUILTIN_TOOLS) == 24
|
||||
assert names[:-1] == [tool["name"] for tool in host.BUILTIN_TOOLS]
|
||||
assert names[-1] == "Workflow"
|
||||
assert handlers["Workflow"] is workflow.run_workflow_sync
|
||||
assert handlers["Workflow"](name="missing") == (
|
||||
"Error: unknown workflow 'missing'"
|
||||
)
|
||||
|
||||
|
||||
def test_workflow_tool_adapter_rejects_model_supplied_code() -> None:
|
||||
workflow = load_lesson(
|
||||
"workflow_schema_test", ROOT / "s18_workflow_runtime" / "code.py"
|
||||
)
|
||||
properties = workflow.WORKFLOW_TOOL["input_schema"]["properties"]
|
||||
|
||||
assert set(properties) == {"name", "args", "resume_from_run_id"}
|
||||
assert "description" not in properties
|
||||
assert "script" not in properties
|
||||
with pytest.raises(workflow.WorkflowInputError, match="name must be a string"):
|
||||
asyncio.run(workflow.run_workflow({"name": "review-changes"}))
|
||||
with pytest.raises(workflow.WorkflowInputError, match="unknown workflow"):
|
||||
asyncio.run(workflow.run_workflow("missing"))
|
||||
|
||||
Reference in New Issue
Block a user