Testing AI Agents Basics

python -m venv .venv && source .venv/bin/activate
pip install pytest

# tools/pricing.py
def quote_total(unit_price: float, qty: int, tax_rate: float = 0.0) -> float:
    if qty < 0:
        raise ValueError("qty must be >= 0")
    return round(unit_price * qty * (1 + tax_rate), 2)
 
 
# tests/test_pricing.py
import pytest
from tools.pricing import quote_total
 
def test_quote_total_happy_path():
    assert quote_total(10.0, 3, tax_rate=0.1) == 33.0
 
def test_quote_total_rejects_negative_qty():
    with pytest.raises(ValueError):
        quote_total(10.0, -1)

from typing import Any
 
REQUIRED = {"title", "priority"}
 
def validate_create_ticket_args(args: dict[str, Any]) -> dict[str, Any]:
    missing = REQUIRED - args.keys()
    if missing:
        raise ValueError(f"missing fields: {sorted(missing)}")
    if args["priority"] not in {"P1", "P2", "P3"}:
        raise ValueError("invalid priority")
    title = str(args["title"]).strip()
    if not title:
        raise ValueError("empty title")
    return {"title": title, "priority": args["priority"]}
 
 
def test_validate_create_ticket_args():
    assert validate_create_ticket_args(
        {"title": "  Outage  ", "priority": "P1"}
    ) == {"title": "Outage", "priority": "P1"}

class FakeHttp:
    def __init__(self, payload):
        self.payload = payload
        self.calls = []
 
    def get(self, path: str):
        self.calls.append(path)
        return self.payload
 
 
def fetch_order(http: FakeHttp, order_id: str) -> dict:
    data = http.get(f"/orders/{order_id}")
    if data.get("status") != "ok":
        raise RuntimeError("order lookup failed")
    return data["order"]
 
 
def test_fetch_order_parses_body():
    http = FakeHttp({"status": "ok", "order": {"id": "42", "total": 19.99}})
    order = fetch_order(http, "42")
    assert order["id"] == "42"
    assert http.calls == ["/orders/42"]

def run_loop(steps, tools):
    log = []
    for step in steps:
        if step["type"] == "tool":
            result = tools[step["name"]](**step["args"])
            log.append({"tool": step["name"], "result": result})
        elif step["type"] == "final":
            log.append({"final": step["text"]})
    return log
 
 
def test_scripted_support_path():
    tools = {
        "lookup_user": lambda email: {"id": "u1", "email": email},
        "create_ticket": lambda **kw: {"id": "t9", **kw},
    }
    steps = [
        {"type": "tool", "name": "lookup_user", "args": {"email": "a@b.com"}},
        {
            "type": "tool",
            "name": "create_ticket",
            "args": {"title": "Help", "user_id": "u1", "priority": "P2"},
        },
        {"type": "final", "text": "Ticket t9 created"},
    ]
    log = run_loop(steps, tools)
    assert [e.get("tool") for e in log if "tool" in e] == [
        "lookup_user",
        "create_ticket",
    ]
    assert log[-1]["final"].startswith("Ticket t9")

GOLD = {
    "id": "refund-basic",
    "input": "Refund order 42",
    "must_call": ["get_order", "issue_refund"],
    "order_id": "42",
}
 
 
def score_run(run: dict, case: dict) -> bool:
    called = set(run.get("tools_called", []))
    if not set(case["must_call"]).issubset(called):
        return False
    return run.get("refund", {}).get("order_id") == case["order_id"]
 
 
def test_gold_case_hard_checks():
    run = {
        "tools_called": ["get_order", "issue_refund"],
        "refund": {"order_id": "42", "status": "ok"},
        "final": "I processed a refund for order 42 in full.",
    }
    assert score_run(run, GOLD) is True

import pytest
from tools.pricing import quote_total
 
@pytest.mark.parametrize(
    "price,qty,tax,expected",
    [
        (0.0, 5, 0.0, 0.0),
        (2.5, 0, 0.2, 0.0),
        (9.99, 1, 0.0, 9.99),
        (1.0, 3, 0.075, 3.23),
    ],
)
def test_quote_total_table(price, qty, tax, expected):
    assert quote_total(price, qty, tax) == expected

class TicketStore:
    def __init__(self):
        self.by_key = {}
 
    def create(self, title: str, idempotency_key: str) -> dict:
        if idempotency_key in self.by_key:
            return self.by_key[idempotency_key]
        ticket = {"id": f"t{len(self.by_key)+1}", "title": title}
        self.by_key[idempotency_key] = ticket
        return ticket
 
 
def test_create_ticket_is_idempotent():
    store = TicketStore()
    a = store.create("Down", idempotency_key="k1")
    b = store.create("Down", idempotency_key="k1")
    assert a == b
    assert len(store.by_key) == 1

def fake_agent(seed: int) -> dict:
    # Stand-in for non-deterministic planner; even seeds "succeed"
    ok = seed % 2 == 0
    return {
        "tools_called": ["get_order", "issue_refund"] if ok else ["get_order"],
        "refund": {"order_id": "42"} if ok else {},
    }
 
 
def test_pass_rate_at_least_half():
    results = [score_run(fake_agent(s), GOLD) for s in range(6)]
    pass_rate = sum(results) / len(results)
    assert pass_rate >= 0.5