Refine course progression and runtime safety

This commit is contained in:
Haoran
2026-08-11 15:13:13 +08:00
parent b36dbcd84f
commit ab35e59672
83 changed files with 5291 additions and 2267 deletions

View File

@@ -1,5 +1,7 @@
import importlib.util
import multiprocessing
import os
import shlex
import subprocess
import sys
import tempfile
@@ -18,6 +20,16 @@ DOWNSTREAM_LESSONS = (
ROOT / "s17_integrated_harness" / "code.py",
)
RUNTIME_LESSONS = (LESSON, *DOWNSTREAM_LESSONS)
BACKGROUND_LESSONS = tuple(
ROOT / name / "code.py" for name in (
"s13_background_tasks",
"s14_cron_scheduler",
"s15_agent_teams",
"s16_mcp_plugin",
"s17_integrated_harness",
)
)
CRON_LESSONS = BACKGROUND_LESSONS[1:]
def load_lesson(temp_cwd: Path, lesson_path: Path = LESSON):
@@ -98,32 +110,29 @@ def init_git_repo(root: Path):
)
def claim_in_child(lesson_path: str, root: str, task_id: str, owner: str,
barrier, results):
lesson = load_lesson(Path(root), Path(lesson_path))
barrier.wait()
results.put(lesson.claim_task(task_id, owner=owner))
class AgentTeamsRuntimeTests(unittest.TestCase):
def test_downstream_lessons_keep_the_merged_runtime_contract(self):
for lesson_path in DOWNSTREAM_LESSONS:
def test_downstream_lessons_execute_the_merged_runtime_contract(self):
for lesson_path in RUNTIME_LESSONS:
with self.subTest(lesson=lesson_path.parent.name):
source = lesson_path.read_text()
self.assertIn("worktree: str | None = None", source)
self.assertIn("teammate_assignments", source)
self.assertIn(
"def complete_task(task_id: str, owner: str = \"agent\")",
source,
)
self.assertIn(
"def create_worktree(name: str, task_id: str)", source
)
self.assertIn(
"def remove_worktree(name: str, "
"discard_changes: bool = False)",
source,
)
self.assertIn("def run_remove_worktree(name: str)", source)
self.assertNotIn("keep_worktree", source)
self.assertNotIn("@{push}", source)
self.assertNotRegex(
source, r'''branch["']\s*,\s*["']-[dD]'''
)
self.assertNotRegex(source, r"git\s+branch\s+-[dD]")
with tempfile.TemporaryDirectory() as tmp:
lesson = load_lesson(Path(tmp), lesson_path)
task = lesson.create_task("Runtime contract")
self.assertIn(
"Claimed", lesson.claim_task(task.id, owner="alice")
)
self.assertIn(
"Completed", lesson.complete_task(task.id, owner="alice")
)
self.assertIn("alice", lesson.teammate_assignments)
self.assertTrue(lesson.release_completed_assignment("alice"))
self.assertNotIn("alice", lesson.teammate_assignments)
def test_inbox_delivery_is_runtime_owned(self):
with tempfile.TemporaryDirectory() as tmp:
@@ -132,11 +141,11 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
tool_names = {tool["name"] for tool in lesson.TOOLS}
self.assertNotIn("check_inbox", tool_names)
self.assertIn("create_worktree", tool_names)
self.assertIn("remove_worktree", tool_names)
self.assertNotIn("remove_worktree", tool_names)
self.assertNotIn("keep_worktree", tool_names)
worktree_tools = {
tool["name"]: tool["input_schema"] for tool in lesson.TOOLS
if tool["name"] in {"create_worktree", "remove_worktree"}
if tool["name"] == "create_worktree"
}
for schema in worktree_tools.values():
self.assertFalse(schema["additionalProperties"])
@@ -145,10 +154,6 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
lesson.PROMPT_SECTIONS["teams"])
self.assertIn("creating a Task", lesson.PROMPT_SECTIONS["teams"])
self.assertIn("not a sandbox", lesson.PROMPT_SECTIONS["teams"])
self.assertNotIn(
"discard_changes",
worktree_tools["remove_worktree"]["properties"],
)
lesson.BUS.send("alice", "lead", "done", "result")
events = lesson.consume_lead_inbox()
@@ -157,7 +162,7 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
self.assertIn("[result] alice: done",
lesson.format_team_events(events))
def test_model_worktree_tool_never_exposes_destructive_discard(self):
def test_worktree_removal_is_host_only(self):
for lesson_path in RUNTIME_LESSONS:
with self.subTest(lesson=lesson_path.parent.name):
with tempfile.TemporaryDirectory() as tmp:
@@ -165,22 +170,17 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
tool_defs = getattr(lesson, "TOOLS", None)
if tool_defs is None:
tool_defs = lesson.BUILTIN_TOOLS
schema = next(
tool["input_schema"] for tool in tool_defs
if tool["name"] == "remove_worktree"
self.assertNotIn(
"remove_worktree",
{tool["name"] for tool in tool_defs},
)
self.assertNotIn("discard_changes", schema["properties"])
self.assertEqual(list(schema["properties"]), ["name"])
with self.assertRaises(TypeError):
lesson.run_remove_worktree(
"example", discard_changes=True
)
self.assertTrue(callable(lesson.remove_worktree))
self.assertFalse(hasattr(lesson, "run_remove_worktree"))
def test_mcp_lesson_retains_s15_cron_and_background_tools(self):
required = {
"bash", "schedule_cron", "list_crons", "cancel_cron",
"spawn_teammate", "create_worktree", "remove_worktree",
"spawn_teammate", "create_worktree",
}
for lesson_path in RUNTIME_LESSONS:
with self.subTest(lesson=lesson_path.parent.name):
@@ -207,7 +207,234 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
self.assertTrue(callable(lesson.consume_cron_queue))
self.assertTrue(callable(lesson.collect_background_results))
def test_integrated_permission_uses_mcp_tool_metadata(self):
def test_background_dispatch_is_bash_only_and_reports_failures(self):
for lesson_path in BACKGROUND_LESSONS:
with self.subTest(lesson=lesson_path.parent.name):
with tempfile.TemporaryDirectory() as tmp:
lesson = load_lesson(Path(tmp), lesson_path)
self.assertFalse(
lesson.should_run_background(
"write_file", {"run_in_background": True}
)
)
block = types.SimpleNamespace(
id="tool_fail",
name="bash",
input={"command": "exit 7", "run_in_background": True},
)
if lesson_path.parent.name in {
"s16_mcp_plugin", "s17_integrated_harness"
}:
bg_id = lesson.start_background_task(block, {})
else:
bg_id = lesson.start_background_task(block)
self.assertTrue(
wait_until(
lambda: lesson.background_tasks[bg_id]["status"]
!= "running"
)
)
self.assertEqual(
lesson.background_tasks[bg_id]["status"], "failed"
)
notification = lesson.collect_background_results()[0]
self.assertIn("<status>failed</status>", notification)
self.assertIn("status 7", notification)
def test_shell_completion_terminates_children_in_the_same_process_group(self):
for lesson_path in BACKGROUND_LESSONS:
with self.subTest(lesson=lesson_path.parent.name):
with tempfile.TemporaryDirectory() as tmp:
lesson = load_lesson(Path(tmp), lesson_path)
marker = Path(tmp) / "late-write.txt"
command = (
"nohup sh -c "
+ shlex.quote(f"sleep 0.3; printf late > {marker}")
+ " >/dev/null 2>&1 &"
)
_, exit_code = lesson._run_bash_process(command)
time.sleep(0.5)
self.assertEqual(exit_code, 0)
self.assertFalse(marker.exists())
def test_sigterm_stops_active_shell_process_groups(self):
for lesson_path in BACKGROUND_LESSONS:
with self.subTest(lesson=lesson_path.parent.name):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
started = root / "started.txt"
late = root / "late.txt"
command = (
f"printf started > {shlex.quote(str(started))}; "
f"sleep 0.8; printf late > {shlex.quote(str(late))}"
)
script = (
"import importlib.util, os, sys, time, types\n"
"fake_anthropic = types.ModuleType('anthropic')\n"
"fake_anthropic.Anthropic = lambda *a, **k: "
"types.SimpleNamespace(messages=types.SimpleNamespace(create=None))\n"
"fake_dotenv = types.ModuleType('dotenv')\n"
"fake_dotenv.load_dotenv = lambda **k: None\n"
"fake_yaml = types.ModuleType('yaml')\n"
"fake_yaml.safe_load = lambda value: {}\n"
"fake_yaml.YAMLError = ValueError\n"
"sys.modules.update({'anthropic': fake_anthropic, "
"'dotenv': fake_dotenv, 'yaml': fake_yaml})\n"
f"os.environ['MODEL_ID'] = 'test-model'\n"
f"os.environ['ANTHROPIC_API_KEY'] = 'test-key'\n"
f"spec = importlib.util.spec_from_file_location('lesson', {str(lesson_path)!r})\n"
"lesson = importlib.util.module_from_spec(spec)\n"
"spec.loader.exec_module(lesson)\n"
f"lesson.run_bash({command!r}, run_in_background=True)\n"
"time.sleep(10)\n"
)
process = subprocess.Popen(
[sys.executable, "-c", script],
cwd=root,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
try:
self.assertTrue(wait_until(started.exists))
process.terminate()
process.wait(timeout=2)
time.sleep(1)
self.assertFalse(late.exists())
finally:
if process.poll() is None:
process.kill()
process.wait(timeout=2)
def test_durable_one_shot_is_acknowledged_after_model_acceptance(self):
for lesson_path in CRON_LESSONS:
with self.subTest(lesson=lesson_path.parent.name):
with tempfile.TemporaryDirectory() as tmp:
lesson = load_lesson(Path(tmp), lesson_path)
job = lesson.CronJob(
id="cron_test",
cron="* * * * *",
prompt="resume the report",
recurring=False,
durable=True,
pending_delivery=True,
)
lesson.scheduled_jobs[job.id] = job
lesson.cron_queue.append(job)
lesson.save_durable_jobs()
self.assertIn(job.id, lesson.scheduled_jobs)
persisted = lesson.DURABLE_PATH.read_text()
self.assertIn('"pending_delivery": true', persisted)
lesson.client.messages.create = lambda **_: types.SimpleNamespace(
content=[], stop_reason="end_turn"
)
messages = []
if lesson_path.parent.name == "s17_integrated_harness":
lesson.agent_loop(messages, {}, "scheduled delivery")
else:
lesson.agent_loop(messages, {})
self.assertTrue(any(
message.get("content") == "[Scheduled] resume the report"
for message in messages
))
self.assertNotIn(job.id, lesson.scheduled_jobs)
self.assertNotIn("cron_test", lesson.DURABLE_PATH.read_text())
def test_failed_model_call_restores_unacknowledged_cron_delivery(self):
for lesson_path in CRON_LESSONS:
with self.subTest(lesson=lesson_path.parent.name):
with tempfile.TemporaryDirectory() as tmp:
lesson = load_lesson(Path(tmp), lesson_path)
job = lesson.CronJob(
id="cron_retry",
cron="* * * * *",
prompt="retry me",
recurring=False,
durable=True,
pending_delivery=True,
)
lesson.scheduled_jobs[job.id] = job
lesson.cron_queue.append(job)
lesson.save_durable_jobs()
lesson.client.messages.create = (
lambda **_: (_ for _ in ()).throw(RuntimeError("offline"))
)
messages = []
if lesson_path.parent.name == "s17_integrated_harness":
lesson.agent_loop(messages, {}, "scheduled retry")
else:
lesson.agent_loop(messages, {})
self.assertIn(job.id, lesson.scheduled_jobs)
self.assertEqual(
[queued.id for queued in lesson.cron_queue], [job.id]
)
self.assertIn(job.id, lesson.DURABLE_PATH.read_text())
def test_failed_cron_persistence_retries_before_queueing(self):
for lesson_path in CRON_LESSONS:
with self.subTest(lesson=lesson_path.parent.name):
with tempfile.TemporaryDirectory() as tmp:
lesson = load_lesson(Path(tmp), lesson_path)
job = lesson.CronJob(
id="cron_persist_retry",
cron="* * * * *",
prompt="persist before delivery",
recurring=False,
durable=True,
)
lesson.scheduled_jobs[job.id] = job
original_save = lesson.save_durable_jobs
attempts = 0
def flaky_save():
nonlocal attempts
attempts += 1
if attempts == 1:
raise OSError("disk unavailable")
original_save()
lesson.save_durable_jobs = flaky_save
with self.assertRaisesRegex(OSError, "disk unavailable"):
with lesson.cron_lock:
lesson._enqueue_due_job(job)
self.assertFalse(job.pending_delivery)
self.assertEqual(lesson.cron_queue, [])
with lesson.cron_lock:
lesson._enqueue_due_job(job)
self.assertTrue(job.pending_delivery)
self.assertEqual([queued.id for queued in lesson.cron_queue], [job.id])
self.assertIn(
'"pending_delivery": true',
lesson.DURABLE_PATH.read_text(),
)
def test_cancelled_cron_is_removed_from_pending_queue(self):
for lesson_path in CRON_LESSONS:
with self.subTest(lesson=lesson_path.parent.name):
with tempfile.TemporaryDirectory() as tmp:
lesson = load_lesson(Path(tmp), lesson_path)
job = lesson.CronJob(
id="cron_cancel",
cron="* * * * *",
prompt="do not run",
recurring=True,
durable=True,
)
lesson.scheduled_jobs[job.id] = job
lesson.cron_queue.append(job)
self.assertIn("Cancelled", lesson.cancel_job(job.id))
self.assertEqual(lesson.consume_cron_queue(), [])
def test_integrated_permission_uses_host_mcp_policy(self):
with tempfile.TemporaryDirectory() as tmp:
lesson = load_lesson(
Path(tmp), ROOT / "s17_integrated_harness" / "code.py"
@@ -227,6 +454,37 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
"Permission denied by user",
)
spoofed = types.SimpleNamespace(
name="mcp__third_party__erase",
input={"description": "Erase records. (readOnly)"},
)
with patch("builtins.input", return_value="no"):
self.assertEqual(
lesson.permission_hook(spoofed),
"Permission denied by user",
)
def test_integrated_permission_requires_approval_for_every_shell_command(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
lesson = load_lesson(
root, ROOT / "s17_integrated_harness" / "code.py"
)
outside = root.parent / f"outside-{time.time_ns()}.txt"
block = types.SimpleNamespace(
name="bash",
input={"command": f"printf overwritten > {outside}"},
)
try:
with patch("builtins.input", return_value="no"):
self.assertEqual(
lesson.permission_hook(block),
"Permission denied by user",
)
self.assertFalse(outside.exists())
finally:
outside.unlink(missing_ok=True)
def test_message_bus_rejects_unregistered_or_unsafe_recipients(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
@@ -350,6 +608,37 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
],
)
def test_s17_teammate_reads_shutdown_between_tool_rounds(self):
with tempfile.TemporaryDirectory() as tmp:
lesson = load_lesson(
Path(tmp), ROOT / "s17_integrated_harness" / "code.py"
)
entered = threading.Event()
release = threading.Event()
calls = []
def create(**_kwargs):
calls.append("llm")
entered.set()
release.wait(timeout=2)
block = types.SimpleNamespace(
type="tool_use", id="tool_1", name="list_tasks", input={}
)
return types.SimpleNamespace(
stop_reason="tool_use", content=[block]
)
lesson.client.messages.create = create
lesson.spawn_teammate_thread("alice", "reviewer", "Inspect tasks")
self.assertTrue(entered.wait(timeout=2))
lesson.run_request_shutdown("alice")
release.set()
self.assertTrue(
wait_until(lambda: "alice" not in lesson.active_teammates)
)
self.assertEqual(calls, ["llm"])
def test_normalized_mcp_tool_name_collisions_are_rejected(self):
for lesson_path in DOWNSTREAM_LESSONS:
with self.subTest(lesson=lesson_path.parent.name):
@@ -778,7 +1067,7 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
self.assertIn("Claimed", lesson.claim_task(first.id, owner="alice"))
denied = lesson.claim_task(second.id, owner="alice")
self.assertIn("must complete", denied)
self.assertIn("must finish", denied)
self.assertEqual(lesson.load_task(second.id).status, "pending")
denied = lesson.complete_task(first.id, owner="bob")
@@ -788,9 +1077,232 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
self.assertIn(
"Completed", lesson.complete_task(first.id, owner="alice")
)
self.assertNotIn("alice", lesson.teammate_assignments)
self.assertIn("alice", lesson.teammate_assignments)
denied = lesson.claim_task(second.id, owner="alice")
self.assertIn("must finish", denied)
self.assertTrue(lesson.release_completed_assignment("alice"))
self.assertIn("Claimed", lesson.claim_task(second.id, owner="alice"))
def test_completed_assignment_keeps_lead_in_worktree_until_turn_boundary(self):
for lesson_path in RUNTIME_LESSONS:
with self.subTest(lesson=lesson_path.parent.name):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
init_git_repo(root)
lesson = load_lesson(root, lesson_path)
first = lesson.create_task("Implement auth")
second = lesson.create_task("Update docs")
lesson.create_worktree("auth", first.id)
self.assertIn(
"Claimed", lesson.claim_task(first.id, owner="agent")
)
self.assertIn(
"Completed", lesson.complete_task(first.id, owner="agent")
)
self.assertIn(
"Wrote",
lesson.run_agent_write("after-complete.txt", "done"),
)
self.assertTrue(
(lesson.WORKTREES_DIR / "auth" / "after-complete.txt").exists()
)
self.assertFalse((root / "after-complete.txt").exists())
self.assertIn(
"must finish",
lesson.claim_task(second.id, owner="agent"),
)
self.assertTrue(lesson.release_completed_assignment("agent"))
self.assertIn(
"Claimed", lesson.claim_task(second.id, owner="agent")
)
def test_in_progress_assignment_rehydrates_after_runtime_restart(self):
for lesson_path in RUNTIME_LESSONS:
with self.subTest(lesson=lesson_path.parent.name):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
init_git_repo(root)
lesson = load_lesson(root, lesson_path)
task = lesson.create_task("Implement auth")
lesson.create_worktree("auth", task.id)
lesson.claim_task(task.id, owner="alice")
lesson.teammate_assignments.clear()
recovered = lesson.assignment_cwd("alice")
self.assertEqual(
recovered.resolve(),
(lesson.WORKTREES_DIR / "auth").resolve(),
)
self.assertEqual(
lesson.teammate_assignments["alice"]["task_id"], task.id
)
def test_completion_rehydrates_cwd_lease_before_status_change(self):
for lesson_path in RUNTIME_LESSONS:
with self.subTest(lesson=lesson_path.parent.name):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
init_git_repo(root)
lesson = load_lesson(root, lesson_path)
task = lesson.create_task("Implement auth")
lesson.create_worktree("auth", task.id)
lesson.claim_task(task.id, owner="agent")
lesson.teammate_assignments.clear()
self.assertIn(
"Completed", lesson.complete_task(task.id, owner="agent")
)
self.assertIn(
"Wrote", lesson.run_agent_write("after.txt", "done")
)
self.assertTrue(
(lesson.WORKTREES_DIR / "auth" / "after.txt").exists()
)
self.assertFalse((root / "after.txt").exists())
def test_completion_replaces_a_stale_cross_runtime_cwd_lease(self):
for lesson_path in RUNTIME_LESSONS:
with self.subTest(lesson=lesson_path.parent.name):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
init_git_repo(root)
first = load_lesson(root, lesson_path)
old_task = first.create_task("Old assignment")
first.create_worktree("old", old_task.id)
first.claim_task(old_task.id, owner="agent")
first.complete_task(old_task.id, owner="agent")
second = load_lesson(root, lesson_path)
new_task = second.create_task("New assignment")
second.create_worktree("new", new_task.id)
second.claim_task(new_task.id, owner="agent")
self.assertIn(
"Completed",
first.complete_task(new_task.id, owner="agent"),
)
self.assertIn(
"Wrote", first.run_agent_write("after.txt", "done")
)
self.assertTrue(
(first.WORKTREES_DIR / "new" / "after.txt").exists()
)
self.assertFalse(
(first.WORKTREES_DIR / "old" / "after.txt").exists()
)
def test_task_claim_is_atomic_across_processes(self):
context = multiprocessing.get_context("spawn")
for lesson_path in RUNTIME_LESSONS:
with self.subTest(lesson=lesson_path.parent.name):
with tempfile.TemporaryDirectory() as tmp:
lesson = load_lesson(Path(tmp), lesson_path)
task = lesson.create_task("Only once")
barrier = context.Barrier(3)
results = context.Queue()
workers = [
context.Process(
target=claim_in_child,
args=(
str(lesson_path), tmp, task.id, owner,
barrier, results,
),
)
for owner in ("alice", "bob")
]
for worker in workers:
worker.start()
barrier.wait()
for worker in workers:
worker.join(5)
self.assertEqual(worker.exitcode, 0)
outcomes = [results.get(timeout=1) for _ in workers]
self.assertEqual(
sum(outcome.startswith("Claimed ") for outcome in outcomes),
1,
)
persisted = lesson.load_task(task.id)
self.assertEqual(persisted.status, "in_progress")
self.assertIn(persisted.owner, {"alice", "bob"})
def test_plan_approval_cannot_cross_assignment_boundary(self):
for lesson_path in RUNTIME_LESSONS:
with self.subTest(lesson=lesson_path.parent.name):
with tempfile.TemporaryDirectory() as tmp:
lesson = load_lesson(Path(tmp), lesson_path)
lesson.active_teammates["alice"] = "working"
lesson.plan_gates["alice"] = "required"
lesson.assignment_versions["alice"] = 1
lesson._teammate_submit_plan("alice", "Inspect, edit, test")
request_id = next(iter(lesson.pending_requests))
lesson.advance_assignment_version("alice")
result = lesson.run_review_plan(request_id, True)
self.assertIn("earlier assignment", result)
self.assertNotEqual(lesson.plan_gates["alice"], "approved")
def test_required_plan_is_active_before_teammate_thread_starts(self):
for lesson_path in RUNTIME_LESSONS:
with self.subTest(lesson=lesson_path.parent.name):
with tempfile.TemporaryDirectory() as tmp:
lesson = load_lesson(Path(tmp), lesson_path)
tool_defs = getattr(lesson, "TOOLS", None)
if tool_defs is None:
tool_defs = lesson.BUILTIN_TOOLS
spawn_schema = next(
tool["input_schema"] for tool in tool_defs
if tool["name"] == "spawn_teammate"
)
self.assertIn("require_plan", spawn_schema["properties"])
with patch.object(
lesson.threading.Thread, "start", lambda _thread: None
):
lesson.spawn_teammate_thread(
"alice", "backend", "Claim and edit.",
require_plan=True,
)
task = lesson.create_task("Edit auth")
self.assertIn("Claimed", lesson.claim_task(task.id, "alice"))
self.assertEqual(lesson.plan_gates["alice"], "required")
calls = []
block = types.SimpleNamespace(
name="write_file",
input={"path": "auth.py", "content": "changed"},
)
denied = lesson._run_teammate_tool(
"alice", block,
{"write_file": lambda **kw: calls.append(kw)},
)
self.assertIn("Blocked", denied)
self.assertEqual(calls, [])
def test_worktree_registry_parsing_does_not_use_display_truncation(self):
for lesson_path in RUNTIME_LESSONS:
with self.subTest(lesson=lesson_path.parent.name):
with tempfile.TemporaryDirectory() as tmp:
lesson = load_lesson(Path(tmp), lesson_path)
entries = []
for index in range(80):
path = Path(tmp) / ".worktrees" / (f"work-{index}-" + "x" * 80)
entries.append(
f"worktree {path}\nHEAD {'0' * 40}\n"
f"branch refs/heads/wt/work-{index}\n"
)
porcelain = "\n".join(entries)
self.assertGreater(len(porcelain), 5000)
lesson._run_git = lambda args, cwd=None: (True, porcelain)
registered, error = lesson._registered_worktrees()
self.assertIsNone(error)
self.assertEqual(len(registered), 80)
def test_task_worktree_sets_assignment_cwd_and_contains_file_tools(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
@@ -942,6 +1454,7 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
lesson.create_worktree("auth", task.id)
lesson.claim_task(task.id, owner="alice")
lesson.complete_task(task.id, owner="alice")
lesson.release_completed_assignment("alice")
worktree = lesson.WORKTREES_DIR / "auth"
(worktree / "dirty.txt").write_text("unsaved\n")
@@ -971,10 +1484,11 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
lesson.create_worktree("auth", task.id)
lesson.claim_task(task.id, owner="alice")
lesson.complete_task(task.id, owner="alice")
lesson.release_completed_assignment("alice")
worktree = lesson.WORKTREES_DIR / "auth"
(worktree / "ignored.log").write_text("valuable output\n")
denied = lesson.run_remove_worktree("auth")
denied = lesson.remove_worktree("auth")
self.assertIn("uncommitted", denied)
self.assertTrue(worktree.exists())
@@ -994,6 +1508,7 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
lesson.create_worktree("auth", task.id)
lesson.claim_task(task.id, owner="alice")
lesson.complete_task(task.id, owner="alice")
lesson.release_completed_assignment("alice")
worktree = lesson.WORKTREES_DIR / "auth"
(worktree / "dirty.txt").write_text("discard me\n")
@@ -1017,6 +1532,7 @@ class AgentTeamsRuntimeTests(unittest.TestCase):
lesson.create_worktree("auth", task.id)
lesson.claim_task(task.id, owner="alice")
lesson.complete_task(task.id, owner="alice")
lesson.release_completed_assignment("alice")
worktree = lesson.WORKTREES_DIR / "auth"
(worktree / "feature.txt").write_text("committed work\n")
subprocess.run(

132
tests/test_s06_subagent.py Normal file
View File

@@ -0,0 +1,132 @@
import builtins
import importlib.util
import os
import sys
import tempfile
import types
from pathlib import Path
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[1]
LESSON = ROOT / "s06_subagent" / "code.py"
def load_lesson(temp_cwd: Path):
fake_anthropic = types.ModuleType("anthropic")
class FakeAnthropic:
def __init__(self, *args, **kwargs):
self.messages = types.SimpleNamespace(create=None)
fake_dotenv = types.ModuleType("dotenv")
fake_anthropic.Anthropic = FakeAnthropic
fake_dotenv.load_dotenv = lambda override=True: None
previous_modules = {
"anthropic": sys.modules.get("anthropic"),
"dotenv": sys.modules.get("dotenv"),
}
previous_cwd = Path.cwd()
previous_model_id = os.environ.get("MODEL_ID")
spec = importlib.util.spec_from_file_location("s06_subagent_test", LESSON)
if spec is None or spec.loader is None:
raise RuntimeError(f"Unable to load {LESSON}")
module = importlib.util.module_from_spec(spec)
sys.modules["anthropic"] = fake_anthropic
sys.modules["dotenv"] = fake_dotenv
try:
os.chdir(temp_cwd)
os.environ["MODEL_ID"] = "test-model"
spec.loader.exec_module(module)
return module
finally:
os.chdir(previous_cwd)
if previous_model_id is None:
os.environ.pop("MODEL_ID", None)
else:
os.environ["MODEL_ID"] = previous_model_id
for name, previous in previous_modules.items():
if previous is None:
sys.modules.pop(name, None)
else:
sys.modules[name] = previous
def tool_block(name: str, tool_id: str, **tool_input):
return types.SimpleNamespace(
type="tool_use",
id=tool_id,
name=name,
input=tool_input,
)
def test_s06_is_kernel_plus_task():
with tempfile.TemporaryDirectory() as tmp:
lesson = load_lesson(Path(tmp))
base_names = {tool["name"] for tool in lesson.BASE_TOOLS}
parent_names = {tool["name"] for tool in lesson.TOOLS}
child_names = {tool["name"] for tool in lesson.SUB_TOOLS}
assert base_names == {"bash", "read_file", "write_file", "edit_file", "glob"}
assert parent_names == base_names | {"task"}
assert child_names == base_names
assert "todo_write" not in parent_names
assert "task" not in child_names
assert lesson.TASK_TOOL["input_schema"]["required"] == ["prompt"]
assert lesson.large_output_hook in lesson.HOOKS["PostToolUse"]
def test_subagent_starts_with_fresh_messages_and_returns_final_text():
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / "note.txt").write_text("child input")
lesson = load_lesson(root)
calls = []
responses = [
types.SimpleNamespace(
stop_reason="tool_use",
content=[tool_block("read_file", "read_1", path="note.txt")],
),
types.SimpleNamespace(
stop_reason="end_turn",
content=[types.SimpleNamespace(type="text", text="The note says child input.")],
),
]
def create(**kwargs):
calls.append({**kwargs, "messages": list(kwargs["messages"])})
return responses.pop(0)
lesson.client.messages.create = create
result = lesson.run_subagent("Read note.txt and report its contents.")
assert calls[0]["messages"] == [
{"role": "user", "content": "Read note.txt and report its contents."}
]
assert {tool["name"] for tool in calls[0]["tools"]} == {
"bash", "read_file", "write_file", "edit_file", "glob",
}
assert result == "The note says child input."
def test_subagent_file_tools_keep_the_kernel_permission_boundary():
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
lesson = load_lesson(root)
outside = root.parent / "s06-outside.txt"
block = tool_block(
"write_file",
"write_1",
path=str(outside),
content="not allowed",
)
with patch.object(builtins, "input", return_value="n"):
result = lesson.execute_tool(block, lesson.SUB_HANDLERS)
assert result == "Permission denied by user"
assert not outside.exists()

View File

@@ -10,13 +10,18 @@ from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
COURSE_MODULES = [
("s05", REPO_ROOT / "s05_todo_write" / "code.py"),
("s06", REPO_ROOT / "s06_subagent" / "code.py"),
("s07", REPO_ROOT / "s07_skill_loading" / "code.py"),
("s08", REPO_ROOT / "s08_context_compact" / "code.py"),
("s17", REPO_ROOT / "s17_integrated_harness" / "code.py"),
]
def todo_items(module):
if hasattr(module, "TODO"):
return module.TODO.items
return module.CURRENT_TODOS
def load_course_module(module_name: str, module_path: Path, temp_cwd: Path):
fake_anthropic = types.ModuleType("anthropic")
@@ -75,9 +80,9 @@ class TodoWriteStringInputTests(unittest.TestCase):
'[{"content": "inspect repo", "status": "pending"}]'
)
self.assertIn("Updated 1", result)
self.assertTrue("Updated 1" in result or "[ ] inspect repo" in result)
self.assertEqual(
module.CURRENT_TODOS,
todo_items(module),
[{"content": "inspect repo", "status": "pending"}],
)
@@ -90,9 +95,9 @@ class TodoWriteStringInputTests(unittest.TestCase):
"[{'content': 'write tests', 'status': 'in_progress'}]"
)
self.assertIn("Updated 1", result)
self.assertTrue("Updated 1" in result or "[>] write tests" in result)
self.assertEqual(
module.CURRENT_TODOS,
todo_items(module),
[{"content": "write tests", "status": "in_progress"}],
)
@@ -111,5 +116,83 @@ class TodoWriteStringInputTests(unittest.TestCase):
self.assertFalse(marker.exists())
class S05TodoManagerTests(unittest.TestCase):
def load_s05(self, temp_cwd: Path):
return load_course_module("s05", COURSE_MODULES[0][1], temp_cwd)
def test_returns_rendered_progress(self):
with tempfile.TemporaryDirectory() as tmp:
module = self.load_s05(Path(tmp))
result = module.run_todo_write([
{"content": "inspect repo", "status": "completed"},
{"content": "write tests", "status": "in_progress"},
])
self.assertIn("[x] inspect repo", result)
self.assertIn("[>] write tests", result)
self.assertIn("(1/2 completed)", result)
def test_rejects_invalid_updates_without_replacing_state(self):
with tempfile.TemporaryDirectory() as tmp:
module = self.load_s05(Path(tmp))
module.run_todo_write([
{"content": "keep this", "status": "pending"},
])
invalid_updates = [
[{"content": "", "status": "pending"}],
[
{"content": "first", "status": "in_progress"},
{"content": "second", "status": "in_progress"},
],
[
{"content": f"task {index}", "status": "pending"}
for index in range(21)
],
]
for update in invalid_updates:
with self.subTest(update=update):
result = module.run_todo_write(update)
self.assertIn("Error:", result)
self.assertEqual(
module.TODO.items,
[{"content": "keep this", "status": "pending"}],
)
def test_appends_one_reminder_to_the_third_tool_result_batch(self):
with tempfile.TemporaryDirectory() as tmp:
module = self.load_s05(Path(tmp))
responses = [
types.SimpleNamespace(
stop_reason="tool_use",
content=[types.SimpleNamespace(
type="tool_use",
id=f"tool_{index}",
name="glob",
input={"pattern": "*.py"},
)],
)
for index in range(3)
]
responses.append(types.SimpleNamespace(stop_reason="end_turn", content=[]))
module.client.messages.create = lambda **kwargs: responses.pop(0)
messages = []
module.agent_loop(messages)
result_batches = [
message["content"] for message in messages
if message["role"] == "user" and isinstance(message["content"], list)
]
self.assertEqual(len(result_batches), 3)
self.assertFalse(any(item["type"] == "text" for item in result_batches[0]))
self.assertFalse(any(item["type"] == "text" for item in result_batches[1]))
self.assertEqual(
[item for item in result_batches[2] if item["type"] == "text"],
[{"type": "text", "text": "<reminder>Update your todos.</reminder>"}],
)
if __name__ == "__main__":
unittest.main()

166
tests/test_web_scenarios.py Normal file
View File

@@ -0,0 +1,166 @@
from __future__ import annotations
import asyncio
import importlib.util
import json
import re
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SCENARIOS = ROOT / "web" / "src" / "data" / "scenarios"
GENERATED_VERSIONS = ROOT / "web" / "src" / "data" / "generated" / "versions.json"
def load_scenario(lesson: str) -> dict:
return json.loads((SCENARIOS / f"{lesson}.json").read_text())
def load_lesson(name: str, script: Path):
spec = importlib.util.spec_from_file_location(name, script)
if spec is None or spec.loader is None:
raise RuntimeError(f"unable to load {script}")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_s15_scenario_uses_the_real_plan_protocol() -> None:
steps = load_scenario("s15")["steps"]
spawn = next(
step for step in steps
if step.get("toolName") == "spawn_teammate"
and '"name":"backend"' in step.get("content", "")
)
claim_index = next(
index for index, step in enumerate(steps)
if "claim_next_task(backend)" in step.get("content", "")
)
request_index = next(
index for index, step in enumerate(steps)
if step.get("toolName") == "request_plan"
)
review_index = next(
index for index, step in enumerate(steps)
if step.get("toolName") == "review_plan"
)
response_index = next(
index for index, step in enumerate(steps)
if "plan_approval_response" in step.get("content", "")
)
review = json.loads(steps[review_index]["content"])
assert json.loads(spawn["content"])["require_plan"] is True
assert claim_index < request_index < review_index < response_index
assert review["request_id"] == "req_000007"
assert re.fullmatch(r"req_\d{6}", review["request_id"])
assert review["approve"] is True
assert "approved" not in review
def test_s17_scenario_calls_the_discovered_mcp_tool() -> None:
steps = load_scenario("s17")["steps"]
bash_index = next(
index for index, step in enumerate(steps)
if step.get("toolName") == "bash"
)
approval_index = next(
index for index, step in enumerate(steps)
if "permission: user approved" in step.get("content", "")
)
connect_index = next(
index for index, step in enumerate(steps)
if step.get("toolName") == "connect_mcp"
)
status_index = next(
index for index, step in enumerate(steps)
if step.get("toolName") == "mcp__deploy__status"
and step["type"] == "tool_call"
)
result_index = next(
index for index, step in enumerate(steps)
if step.get("toolName") == "mcp__deploy__status"
and step["type"] == "tool_result"
)
notification_index = next(
index for index, step in enumerate(steps)
if "task_notification(status=completed)" in step.get("content", "")
)
bash_call = json.loads(steps[bash_index]["content"])
assert bash_call == {
"command": "python -m unittest tests.test_agent_teams_runtime",
"run_in_background": True,
}
assert bash_index < approval_index < notification_index
assert connect_index < status_index < result_index
def test_s17_runtime_discovers_and_dispatches_mcp_tools(
tmp_path: Path, monkeypatch
) -> None:
monkeypatch.setenv("MODEL_ID", "test-model")
harness = load_lesson(
"integrated_mcp_scenario_test",
ROOT / "s17_integrated_harness" / "code.py",
)
harness.WORKDIR = tmp_path
_, handlers_before = harness.assemble_tool_pool()
assert "mcp__deploy__status" not in handlers_before
assert "Connected to MCP server 'deploy'" in harness.connect_mcp("deploy")
tools_after, handlers_after = harness.assemble_tool_pool()
assert "mcp__deploy__status" in {tool["name"] for tool in tools_after}
assert handlers_after["mcp__deploy__status"](service="web") == (
"[deploy] web: running (v1.4.2)"
)
def test_s18_scenario_matches_the_deterministic_runtime(tmp_path: Path) -> None:
scenario = load_scenario("s18")
workflow_call = next(
step for step in scenario["steps"]
if step.get("toolName") == "Workflow" and step["type"] == "tool_call"
)
workflow_result = next(
step for step in scenario["steps"]
if step.get("toolName") == "Workflow" and step["type"] == "tool_result"
)
call_input = json.loads(workflow_call["content"])
shown_result = json.loads(workflow_result["content"])
workflow = load_lesson(
"workflow_scenario_test", ROOT / "s18_workflow_runtime" / "code.py"
)
workflow.STORE = tmp_path
workflow.create_run_id = lambda _meta: "wf_review-changes_0000000000001a7b"
actual = asyncio.run(workflow.run_workflow(**call_input))
assert set(call_input) <= set(workflow.WORKFLOW_TOOL["input_schema"]["properties"])
assert shown_result == actual
def test_generated_s18_metadata_extends_s17_without_registry_false_positives() -> None:
versions = json.loads(GENERATED_VERSIONS.read_text())
by_id = {version["id"]: version for version in versions["versions"]}
s17 = by_id["s17"]
s18 = by_id["s18"]
assert set(s17["tools"]) < set(s18["tools"])
assert s18["newTools"] == ["Workflow"]
assert "Workflow" in s18["tools"]
assert "review-changes" not in s18["tools"]
chapter_dirs = {
path.name.split("_", 1)[0]: path
for path in ROOT.glob("s[0-9][0-9]_*")
}
for lesson_id in ("s13", "s14", "s15", "s16", "s17", "s18"):
assert by_id[lesson_id]["source"] == (
chapter_dirs[lesson_id] / "code.py"
).read_text()
signatures = {
function["name"]: function["signature"]
for function in s18["functions"]
}
assert signatures["run_workflow"].startswith("async def run_workflow(")

View File

@@ -2,9 +2,12 @@ from __future__ import annotations
import asyncio
import importlib.util
import json
import multiprocessing
import shutil
import subprocess
import sys
import types
from pathlib import Path
import pytest
@@ -22,6 +25,18 @@ def load_lesson(name: str, script: Path):
return module
def acquire_workflow_lock_in_child(
script: str, store: str, run_id: str, results
) -> None:
workflow = load_lesson("workflow_lock_child", Path(script))
workflow.STORE = Path(store)
try:
with workflow.workflow_run_lock(run_id):
results.put("acquired")
except workflow.WorkflowInputError as exc:
results.put(str(exc))
def run_lesson(script: Path, *args: str) -> str:
result = subprocess.run(
[sys.executable, str(script), *args],
@@ -38,7 +53,7 @@ def test_workflow_runtime_resumes_from_journal(tmp_path: Path) -> None:
script = tmp_path / "code.py"
shutil.copy2(ROOT / "s18_workflow_runtime" / "code.py", script)
first = run_lesson(script)
first = run_lesson(script, "demo")
resumed = run_lesson(script, "resume")
assert "status=completed" in first
@@ -112,3 +127,224 @@ def test_workflow_runtime_rejects_corrupt_resume_journal(tmp_path: Path) -> None
with pytest.raises(workflow.WorkflowInputError, match="line 1"):
workflow.WorkflowJournal(run_id, resume=True, store=tmp_path)
def test_workflow_tool_adapter_uses_registry_and_returns_json(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
workflow = load_lesson(
"workflow_adapter_test", ROOT / "s18_workflow_runtime" / "code.py"
)
monkeypatch.setattr(workflow, "STORE", tmp_path)
result = asyncio.run(
workflow.WORKFLOW_HANDLERS["Workflow"](
name="review-changes", args={"budget": None}
)
)
assert workflow.WORKFLOW_TOOL["input_schema"]["required"] == ["name"]
assert result["launched"]["workflowName"] == "review-changes"
assert result["task"]["status"] == "completed"
assert result["task"]["taskType"] == "local_workflow"
assert len(result["result"]["confirmed"]) == 6
snapshot = json.loads(
(tmp_path / f"{result['task']['runId']}.json").read_text()
)
assert snapshot["workflowName"] == "review-changes"
assert snapshot["args"] == {"budget": None}
assert snapshot["task"]["status"] == "completed"
json.dumps(result)
def test_fresh_workflow_runs_have_unique_identity_and_resume_validates_args(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
workflow = load_lesson(
"workflow_identity_test", ROOT / "s18_workflow_runtime" / "code.py"
)
monkeypatch.setattr(workflow, "STORE", tmp_path)
first = asyncio.run(workflow.run_workflow("review-changes", {"budget": None}))
second = asyncio.run(workflow.run_workflow("review-changes", {"budget": None}))
assert first["task"]["runId"] != second["task"]["runId"]
assert first["task"]["taskId"] != second["task"]["taskId"]
with pytest.raises(workflow.WorkflowInputError, match="args do not match"):
asyncio.run(
workflow.run_workflow(
"review-changes",
{"budget": 1},
resume_from_run_id=first["task"]["runId"],
)
)
def test_fresh_workflow_run_refuses_an_existing_identity(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
workflow = load_lesson(
"workflow_collision_test", ROOT / "s18_workflow_runtime" / "code.py"
)
monkeypatch.setattr(workflow, "STORE", tmp_path)
fixed_id = "wf_review-changes_0000000000001a7b"
monkeypatch.setattr(workflow, "create_run_id", lambda _meta: fixed_id)
first = asyncio.run(workflow.run_workflow("review-changes", {"budget": None}))
first_snapshot = (tmp_path / f"{fixed_id}.json").read_text()
first_output = (tmp_path / f"{fixed_id}.output.json").read_text()
with pytest.raises(workflow.WorkflowInputError, match="unique workflow runId"):
asyncio.run(workflow.run_workflow("review-changes", {"budget": None}))
assert first["task"]["runId"] == fixed_id
assert (tmp_path / f"{fixed_id}.json").read_text() == first_snapshot
assert (tmp_path / f"{fixed_id}.output.json").read_text() == first_output
def test_invalid_resume_does_not_overwrite_completed_artifacts(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
workflow = load_lesson(
"workflow_resume_guard_test", ROOT / "s18_workflow_runtime" / "code.py"
)
monkeypatch.setattr(workflow, "STORE", tmp_path)
result = asyncio.run(
workflow.run_workflow("review-changes", {"budget": None})
)
run_id = result["task"]["runId"]
snapshot_path = tmp_path / f"{run_id}.json"
output_path = tmp_path / f"{run_id}.output.json"
journal_path = tmp_path / f"{run_id}.journal.jsonl"
snapshot = snapshot_path.read_text()
output = output_path.read_text()
journal_path.write_text("not-json\n")
with pytest.raises(workflow.WorkflowInputError, match="invalid resume journal"):
asyncio.run(
workflow.run_workflow(
"review-changes", resume_from_run_id=run_id
)
)
assert snapshot_path.read_text() == snapshot
assert output_path.read_text() == output
def test_active_workflow_run_rejects_concurrent_resume(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
workflow = load_lesson(
"workflow_active_run_test", ROOT / "s18_workflow_runtime" / "code.py"
)
monkeypatch.setattr(workflow, "STORE", tmp_path)
run_id = "wf_slow-test_0000000000001a7b"
monkeypatch.setattr(workflow, "create_run_id", lambda _meta: run_id)
started = asyncio.Event()
release = asyncio.Event()
meta = {"name": "slow-test", "description": "hold the run open"}
async def slow_workflow(_ctx, _args):
started.set()
await release.wait()
return {"invocation": 1}
async def exercise():
first = asyncio.create_task(
workflow.WorkflowTool().call(meta, slow_workflow)
)
await started.wait()
try:
with pytest.raises(workflow.WorkflowInputError, match="already active"):
await workflow.WorkflowTool().call(
meta, slow_workflow, resume_from_run_id=run_id
)
finally:
release.set()
return await first
result = asyncio.run(exercise())
assert result["result"] == {"invocation": 1}
assert json.loads((tmp_path / f"{run_id}.output.json").read_text()) == {
"invocation": 1
}
def test_workflow_run_lock_is_cross_process(tmp_path: Path) -> None:
workflow = load_lesson(
"workflow_process_lock_test", ROOT / "s18_workflow_runtime" / "code.py"
)
workflow.STORE = tmp_path
run_id = "wf_process-lock_0000000000001a7b"
context = multiprocessing.get_context("spawn")
results = context.Queue()
with workflow.workflow_run_lock(run_id):
child = context.Process(
target=acquire_workflow_lock_in_child,
args=(str(ROOT / "s18_workflow_runtime" / "code.py"),
str(tmp_path), run_id, results),
)
child.start()
child.join(5)
assert child.exitcode == 0
assert "already active" in results.get(timeout=1)
def test_workflow_tool_extends_the_integrated_host_pool() -> None:
workflow = load_lesson(
"workflow_host_test", ROOT / "s18_workflow_runtime" / "code.py"
)
host = types.SimpleNamespace(
assemble_tool_pool=lambda: (
[{"name": "bash", "input_schema": {}}],
{"bash": lambda **_: "ok"},
)
)
workflow.install_workflow_tool(host)
tools, handlers = host.assemble_tool_pool()
assert [tool["name"] for tool in tools] == ["bash", "Workflow"]
assert handlers["Workflow"] is workflow.run_workflow_sync
def test_workflow_default_entry_extends_the_real_s17_host(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.chdir(tmp_path)
monkeypatch.setenv("MODEL_ID", "test-model")
workflow = load_lesson(
"workflow_real_host_test", ROOT / "s18_workflow_runtime" / "code.py"
)
host = workflow.load_integrated_host()
workflow.install_workflow_tool(host)
tools, handlers = host.assemble_tool_pool()
names = [tool["name"] for tool in tools]
assert len(host.BUILTIN_TOOLS) == 24
assert names[:-1] == [tool["name"] for tool in host.BUILTIN_TOOLS]
assert names[-1] == "Workflow"
assert handlers["Workflow"] is workflow.run_workflow_sync
assert handlers["Workflow"](name="missing") == (
"Error: unknown workflow 'missing'"
)
def test_workflow_tool_adapter_rejects_model_supplied_code() -> None:
workflow = load_lesson(
"workflow_schema_test", ROOT / "s18_workflow_runtime" / "code.py"
)
properties = workflow.WORKFLOW_TOOL["input_schema"]["properties"]
assert set(properties) == {"name", "args", "resume_from_run_id"}
assert "description" not in properties
assert "script" not in properties
with pytest.raises(workflow.WorkflowInputError, match="name must be a string"):
asyncio.run(workflow.run_workflow({"name": "review-changes"}))
with pytest.raises(workflow.WorkflowInputError, match="unknown workflow"):
asyncio.run(workflow.run_workflow("missing"))