Agent tasks with their own checks

from ezvals import eval, EvalContext

ezvals_defaults = {"target": run_agent, "dataset": "agent_tasks"}

@eval(input="What's the weather in Tokyo?")
async def test_weather(ctx: EvalContext):
    actual = await weather_api.current("Tokyo")
    ctx.reference = f"{actual.temp}°F, {actual.condition}"
    assert str(actual.temp) in ctx.output, "Should mention the temperature"

@eval(input="What's Apple's stock price?")
async def test_stock_price(ctx: EvalContext):
    actual = await stock_api.price("AAPL")
    ctx.reference = actual
    assert abs(extract_price(ctx.output) - actual) < 5, "Price too far from the live price"

@eval(input="Book a meeting with John tomorrow at 2pm")
async def test_booking(ctx: EvalContext):
    booking = await calendar_api.find(attendee="john", date="tomorrow")
    ctx.store(metadata={"booking": booking})
    assert booking is not None, "No booking in the calendar"
    assert booking.time == "14:00", f"Booked at {booking.time}"

@eval(input="What's the factorial of 10?", reference="3628800")
async def test_factorial(ctx: EvalContext):
    assert ctx.reference in ctx.output

import assert from "node:assert";
import { evaluate } from "ezvals";

export const ezvalsDefaults = { target: runAgent, dataset: "agent_tasks" };

evaluate("test_weather", { input: "What's the weather in Tokyo?" }, async (ctx) => {
  const actual = await weatherApi.current("Tokyo");
  ctx.reference = `${actual.temp}°F, ${actual.condition}`;
  assert(String(ctx.output).includes(String(actual.temp)), "Should mention the temperature");
});

evaluate("test_stock_price", { input: "What's Apple's stock price?" }, async (ctx) => {
  const actual = await stockApi.price("AAPL");
  ctx.reference = actual;
  assert(Math.abs(extractPrice(String(ctx.output)) - actual) < 5, "Price too far from the live price");
});

evaluate("test_booking", { input: "Book a meeting with John tomorrow at 2pm" }, async (ctx) => {
  const booking = await calendarApi.find({ attendee: "john", date: "tomorrow" });
  ctx.store({ metadata: { booking } });
  assert(booking, "No booking in the calendar");
  assert.equal(booking.time, "14:00", `Booked at ${booking.time}`);
});

evaluate("test_factorial", { input: "What's the factorial of 10?", reference: "3628800" }, (ctx) => {
  assert(String(ctx.output).includes(String(ctx.reference)));
});