원격 Scorer로 에이전트 턴 점수화하기

{
  "schema_version": 2,
  "scoring_target": {
    "type": "agent_turn",
    "schema_version": 1,
    "payload": {
      "event_type": "weave.genai.turn_ended",
      "project_id": "[YOUR-TEAM]/[YOUR-PROJECT]",
      "trace_id": "0af7651916cd43dd8448eb211c80319c",
      "span_id": "b7ad6b7169203331",
      "span_name": "invoke_agent sample-support-agent",
      "operation_name": "invoke_agent",
      "started_at": "2026-06-08T12:00:00+00:00",
      "ended_at": "2026-06-08T12:00:01+00:00",
      "conversation": {
        "id": "conversation-0001",
        "name": "Support session"
      },
      "agent": {
        "name": "sample-support-agent",
        "version": "2026-06-08",
        "description": "Answers support questions"
      },
      "status": {
        "code": "UNSET",
        "message": null,
        "error_type": null
      },
      "messages": {
        "system_instructions": [
          "Answer the user accurately and concisely."
        ],
        "input": [
          {
            "role": "user",
            "content": "What are your support hours?",
            "finish_reason": ""
          }
        ],
        "output": [
          {
            "role": "assistant",
            "content": "Our support team is available Monday through Friday, 9am to 5pm Eastern.",
            "finish_reason": "stop"
          }
        ]
      }
    }
  },
  "scoring_call_id": "018f8d6c-8d5f-7000-8000-000000000005",
  "scoring_trace_id": "018f8d6c-8d5f-7000-8000-000000000006",
  "monitor": {
    "name": "example_remote_scorer_agent_monitor",
    "version_digest": "monitor-version-digest"
  },
  "scorer": {
    "name": "example_remote_scorer",
    "ref": "weave:///[YOUR-TEAM]/[YOUR-PROJECT]/object/example_remote_scorer:scorer-version-digest",
    "config": {
      "example_threshold": 0.8
    }
  },
  "triggered_at": "2026-06-08T12:00:02+00:00"
}

{
  "schema_version": 2,
  "result": {
    "value": "concise",
    "reason": "72 characters.",
    "confidence": 0.9
  }
}

import weave
from weave.flow.monitor import Monitor
from weave.scorers.remote_scorer import RemoteScorer, StaticBearerAuthConfig

weave.init("[YOUR-TEAM]/[YOUR-PROJECT]")

scorer = RemoteScorer(
    name="policy_remote_scorer",
    endpoint_url="https://scoring.example.com/weave/score",
    config={"threshold": 0.9},  # 엔드포인트에 scorer.config로 전달됨
    auth_config=StaticBearerAuthConfig(
        mode="static_bearer",
        bearer_secret_name="WEAVE_REMOTE_SCORER_BEARER_TOKEN",
    ),
)
weave.publish(scorer, name="policy_remote_scorer")

monitor = Monitor(
    name="policy_remote_signal",
    scorers=[scorer],
    op_names=["weave.genai.turn_ended"],  # 완료된 에이전트 턴을 점수화함
    sampling_rate=1.0,
)
monitor.activate()

python3 -m venv .venv && source .venv/bin/activate
python -m pip install -r requirements.txt
export REMOTE_SCORER_DEV_BEARER_TOKEN="dev-token"
uvicorn remote_scorer_app:app --host 127.0.0.1 --port 8000

curl -sS http://127.0.0.1:8000/score \
  -H "Authorization: Bearer dev-token" \
  -H "Content-Type: application/json" \
  -H "Idempotency-Key: local-contract-check" \
  -H "X-Correlation-ID: local-contract-check" \
  -H "X-Weave-Schema-Version: 2" \
  --data @sample_request_v2_agent_turn.json

import uuid

from opentelemetry import trace

import weave
from weave.conversation import Message, log_turn

weave.init("[YOUR-TEAM]/[YOUR-PROJECT]")

result = log_turn(
    conversation_id=f"sample-{uuid.uuid4().hex}",
    agent_name="sample-support-agent",
    system_instructions=["Answer the user accurately and concisely."],
    messages=[Message.user("What are your support hours?")],
    output_messages=[
        Message.assistant(
            "Our support team is available Monday through Friday, 9am to 5pm Eastern."
        )
    ],
)

# weave.init은 백그라운드 스레드에서 span을 내보냅니다. 짧은 스크립트에서는 종료되기 전에 플러시하세요.
trace.get_tracer_provider().force_flush(30_000)
print(result.conversation_id, result.trace_ids)