File-Level Defaults

# evals/customer_service.py

ezvals_defaults = {
    "labels": ["production"],
    "default_score_key": "quality",
}

@eval(input="I want a refund")
async def test_refund(ctx: EvalContext):
    ctx.output = await agent(ctx.input)
    assert "refund" in ctx.output.lower(), "Should acknowledge refund"

@eval(input="Your product is broken")
async def test_complaint(ctx: EvalContext):
    ctx.output = await agent(ctx.input)
    assert "sorry" in ctx.output.lower(), "Should express empathy"

ezvals_defaults = {
    "dataset": "customer_service",
    "labels": ["production"],
    "default_score_key": "quality",
}

@eval  # Uses all defaults
async def test_standard(ctx: EvalContext):
    ...

@eval(labels=["experimental"])  # Override labels only
async def test_new_feature(ctx: EvalContext):
    ...

@eval(dataset="edge_cases", labels=["debug"])  # Override multiple
async def test_edge_case(ctx: EvalContext):
    ...

ezvals_defaults = {
    "metadata": {"environment": "test", "version": "1.0"}
}

@eval(metadata={"category": "refunds"})
async def test_refund(ctx: EvalContext):
    # metadata = {"environment": "test", "version": "1.0", "category": "refunds"}
    ...

def file_evaluator(result):
    return {"key": "file_check", "passed": True}

def specific_evaluator(result):
    return {"key": "specific_check", "passed": True}

ezvals_defaults = {
    "evaluators": [file_evaluator]
}

@eval  # Uses file_evaluator from defaults
async def test_uses_default(ctx: EvalContext):
    ...

@eval(evaluators=[specific_evaluator])  # REPLACES file_evaluator
async def test_with_override(ctx: EvalContext):
    # Only specific_evaluator runs, not file_evaluator
    ...

@eval(evaluators=[file_evaluator, specific_evaluator])
async def test_with_both(ctx: EvalContext):
    ...

evals/
├── customer_service.py    # dataset="customer_service"
├── technical_support.py   # dataset="technical_support"
├── sales.py               # dataset="sales"
└── edge_cases.py          # dataset="edge_cases"

# evals/technical_support.py

ezvals_defaults = {
    "dataset": "technical_support",
    "labels": ["production", "technical"],
    "default_score_key": "helpfulness",
    "metadata": {"department": "engineering"},
    "timeout": 30.0,  # Technical queries may take longer
}

@eval
async def test_debug_help(ctx: EvalContext):
    ...

@eval
async def test_installation_guide(ctx: EvalContext):
    ...

# evals/sentiment_analysis.py

from ezvals import eval, EvalContext

# Shared evaluators
def check_confidence(result):
    if result.metadata and 'confidence' in result.metadata:
        conf = result.metadata['confidence']
        return {
            "key": "high_confidence",
            "passed": conf > 0.8,
            "notes": f"Confidence: {conf:.2f}"
        }
    return None

# File-level defaults
ezvals_defaults = {
    "labels": ["ml", "nlp"],
    "default_score_key": "accuracy",
    "metadata": {"model_version": "v2.1"},
    "evaluators": [check_confidence],
}

# All evals inherit defaults
@eval(cases=[
    {"input": "I love this!", "reference": "positive"},
    {"input": "This is terrible", "reference": "negative"},
])
async def test_basic_sentiment(ctx: EvalContext):
    result = await analyze(ctx.input)
    ctx.output = result["label"]
    ctx.metadata["confidence"] = result["confidence"]
    assert ctx.output == ctx.reference

@eval(input="Oh great, another meeting", labels=["ml", "nlp", "edge_cases"])
async def test_sarcasm(ctx: EvalContext):
    result = await analyze(ctx.input)
    ctx.output = result["label"]
    assert ctx.output == "negative", "Should detect sarcasm as negative"