Checks

evals/cases/<name>.yaml     one scripted check of this harness
evals/graders/<name>.py     the judges those cases call
evals/results/              artifacts a run left behind

# evals/cases/tone.yaml
defaults: {trials: 3, timeout: 300s}

scenarios:
  - id: answers-in-one-line
    user: What is the capital of France?
    expect:
      - final_answer_contains: Paris
      - final_answer_max_chars: 120
      - no_error: true

  - id: uses-the-shell-rather-than-guessing
    user:
      - List the files in the home directory.
      - Now count them.
    trials: 5
    expect:
      - tool_called: {name: shell__run}
      - min_tool_calls: {name: shell__run, count: 2}
      - grader: {name: quality.answers, pass_below: 0.7}

harness_eval

# evals/graders/quality.py

import re


def answers(t, case, run):
    """Did the answer actually answer, and in the register we want?"""
    verdict = t.llm(
        "Answer with a number from 0 to 1 and one sentence of why.\n\n"
        f"Question: {case['user'][0]}\n"
        f"Answer: {run['final_answer']}"
    )
    found = re.search(r"\d+(?:\.\d+)?", verdict)
    score = float(found.group()) if found else 0.0
    return {"pass": score >= 0.7, "score": score, "why": verdict}

case = {"id": ..., "user": [...], "expect": [...]}
run  = {"final_answer": ..., "tool_calls": [...], "error": ..., "turns": ..., "duration_ms": ...}