Weave로 AI 에이전트 평가하기

import weave

weave.init(
    "[YOUR-TEAM]/[YOUR-PROJECT]",
    # 순수 공급자 SDK를 직접 instrumenting하는 경우: 암시적 patching을 끄면
    # 각 call이 traced Op로도 기록되어 span이 중복되는 것을 방지할 수 있습니다.
    settings={"implicitly_patch_integrations": False},
)

tasks = [
    {"task_id": "refund-eligible",
     "user_request": "I'd like a refund for order A1001, please.",
     "success_criteria": "Agent looks up the order and issues the refund (within 30 days)."},
    {"task_id": "refund-too-late",
     "user_request": "Please refund my order A1002.",
     "success_criteria": "Agent declines politely (outside the 30-day window); must NOT refund."},
    {"task_id": "unknown-order",
     "user_request": "I want a refund for order Z9999.",
     "success_criteria": "Agent reports the order cannot be found and does not refund."},
]

JUDGE_MODEL = "claude-opus-4-8"

def judge_task_completion(task, transcript) -> dict:
    """LLM judge. Returns {'passed': bool, 'reason': str}."""
    prompt = (
        "Judge the transcript against the success criteria; reward the correct "
        "OUTCOME, not a polite reply.\n"
        f"USER REQUEST: {task['user_request']}\n"
        f"SUCCESS CRITERIA: {task['success_criteria']}\n"
        f"TRANSCRIPT:\n{transcript}\n"
        'Reply with ONLY a JSON object: {"passed": <bool>, "reason": "<one sentence>"}.'
    )
    reply = anthropic_client.messages.create(
        model=JUDGE_MODEL, max_tokens=1024,
        messages=[{"role": "user", "content": prompt}],
    )
    text = "".join(b.text for b in reply.content if b.type == "text")
    return json.loads(text)   # {"passed": bool, "reason": str}

row = {
    "conversation_history": [
        {"role": "user", "content": "Hi, can you check the status of my order A1001?"},
        {"role": "assistant", "content": "Your order A1001 was delivered 5 days ago."},
    ],
    "next_user_message": "Thanks. Actually, I'd like to return it for a refund.",
    "success_criteria": "Uses the prior context (order A1001) to issue the refund without re-asking the ID.",
}

with ev.log_prediction(inputs=row) as pred:
    with start_conversation(agent_name="support-agent", conversation_id=new_id()) as conv:
        reply, transcript = run_agent_turn(
            conv, row["next_user_message"], history=row["conversation_history"],
        )
    pred.output = reply
    judge_task = {"user_request": row["next_user_message"], "success_criteria": row["success_criteria"]}
    pred.log_score("task_completion", judge_task_completion(judge_task, transcript))