evaluation:
  dataset: eval/input/tasks.json
  output_dir: eval/output
  agent_url: http://127.0.0.1:9900
  agent_startup_timeout_s: 45
  agent_shutdown_timeout_s: 10
  # Number of times each task is replayed. Keep 1 for a first run; raise it
  # (e.g. 5) to measure consistency / pass@k on a non-deterministic agent.
  k: 1
  # Phase 1: deterministic checks only (status, output_must_contain, tool_calls).
  # Switch to true for Phase 2 to also score answers with the LLM judge below.
  qualitative: false

# Used only when qualitative: true. The judge reads expected_outcome from each
# task and scores the agent's final answer against it.
judge:
  provider: openai
  model: gpt-4.1-mini
  api_key_env: OPENAI_API_KEY

# Models the SLA Agent is evaluated with. One run is produced per model, so you
# can compare how the same dataset scores across different LLMs.
models:
  - provider: openai
    model: gpt-4.1-mini