LLM judge

from ezvals import eval, EvalContext

JUDGE_PROMPT = """You grade replies from a customer support agent.
Pass only if the reply is polite, answers the question and makes no promises
the policy doesn't allow. Reply with PASS or FAIL, then one sentence explaining why.

Question: {input}
Reply: {output}"""

async def policy_judge(result):
    verdict = await call_llm(JUDGE_PROMPT.format(input=result.input, output=result.output))
    return {"key": "policy", "passed": verdict.startswith("PASS"), "notes": verdict}

@eval(input="Can I get a refund after 45 days?", target=run_support_agent, evaluators=[policy_judge])
async def test_late_refund(ctx: EvalContext):
    assert "30 days" in ctx.output, "Should state the 30-day window"

import assert from "node:assert";
import { evaluate, type EvalResult } from "ezvals";

const JUDGE_PROMPT = `You grade replies from a customer support agent.
Pass only if the reply is polite, answers the question and makes no promises
the policy doesn't allow. Reply with PASS or FAIL, then one sentence explaining why.`;

async function policyJudge(result: EvalResult) {
  const verdict = await callLlm(`${JUDGE_PROMPT}\n\nQuestion: ${result.input}\nReply: ${result.output}`);
  return { key: "policy", passed: verdict.startsWith("PASS"), notes: verdict };
}

evaluate("test_late_refund", {
  input: "Can I get a refund after 45 days?",
  target: runSupportAgent,
  evaluators: [policyJudge],
}, (ctx) => {
  assert(String(ctx.output).includes("30 days"), "Should state the 30-day window");
});