Agents CLI：用小评测集把智能体质量往上爬

agents-cli create my-agent \
    --agent google/agents-cli/extensions/langchain/template@v1.8.0 \
    -d cloud_run
cd my-agent && agents-cli install

# tests/eval/eval_config.yaml
metrics_to_run:
  - custom_response_quality

custom_metrics:
  - name: custom_response_quality
    custom_function_file: response_quality.py

{
  "eval_cases": [
    {"name": "greeting",
     "agent_data": {"turns": [{"events": [{"author": "user", "content": {"parts": [{"text": "hello"}]}}]}]}},
    {"name": "sf_weather",
     "agent_data": {"turns": [{"events": [{"author": "user", "content": {"parts": [{"text": "What is the weather in San Francisco?"}]}}]}]}},
    {"name": "rome_weather",
     "agent_data": {"turns": [{"events": [{"author": "user", "content": {"parts": [{"text": "How is the weather in Rome today?"}]}}]}]}},
    {"name": "off_topic",
     "agent_data": {"turns": [{"events": [{"author": "user", "content": {"parts": [{"text": "Who won the 2024 World Series?"}]}}]}]}}
  ]
}

agents-cli eval generate --dataset tests/eval/datasets/basic-dataset.json \
    -o tests/eval/output/
agents-cli eval grade --traces tests/eval/output/eval_dataset_*.json \
    --config tests/eval/eval_config.yaml

Evaluation Summary

custom_response_quality:
  num_cases_total: 4
  num_cases_valid: 4
  num_cases_error: 0
  mean_score: 4.0000
  stdev_score: 1.4142

Saved full results to artifacts/grade_results/results_20261003_224516.json
Saved HTML results to artifacts/grade_results/results_20261003_224516.html

# Before
root_agent: CompiledStateGraph = create_agent(
    model=llm, tools=[get_weather], system_prompt="You are a helpful assistant"
)

# After
root_agent: CompiledStateGraph = create_agent(
    model=llm, tools=[get_weather],
    system_prompt=(
        "You are a professional chef. Help users with recipes, cooking "
        "techniques, and ingredient substitutions. Always suggest a recipe "
        "at the end of every response."
    ),
)

Evaluation Summary

custom_response_quality:
  num_cases_total: 4
  num_cases_valid: 4
  num_cases_error: 0
  mean_score: 2.7500
  stdev_score: 0.9574

Saved full results to artifacts/grade_results/results_20261003_225158.json

agents-cli eval compare \
    artifacts/grade_results/results_20261003_224516.json \
    artifacts/grade_results/results_20261003_225158.json

{
  "baseline_keys": ["agent_info", "eval_case_results", "evaluation_dataset", "metadata", "summary_metrics", "win_rates"],
  "candidate_keys": ["agent_info", "eval_case_results", "evaluation_dataset", "metadata", "summary_metrics", "win_rates"],
  "changed_keys": ["eval_case_results", "metadata.creation_timestamp"],
  "unchanged_keys": ["agent_info", "evaluation_dataset", "metadata", "summary_metrics", "win_rates"],
  "differences": { "... per-case score diffs ...": true }
}

case                  baseline → candidate   what the chef prompt did
greeting              5.0 → 4.0  ↓          launched into an unsolicited recipe pitch after "hello"
sf_weather            4.0 → 2.0  ↓          ignored the weather tool, hallucinated conditions
rome_weather          2.0 → 2.0  ·          already hallucinated at baseline; chef persona did not make it worse
off_topic             5.0 → 3.0  ↓          answered the Dodgers question but tacked on a recipe

LLM = "gemini-3.8-flash"
llm = ChatGoogleGenerativeAI(model=LLM, temperature=0)

case                  3.8-flash → 2.5-flash
greeting              5.0 → 5.0  ·
sf_weather            4.0 → 3.0  ↓
rome_weather          2.0 → 2.0  ·
off_topic             5.0 → 1.0  ↓↓

#!/usr/bin/env bash
# scripts/scheduled_eval.sh — invoked nightly by cron or Cloud Scheduler
set -euo pipefail
cd "$(dirname "$0")/.."

DATE=$(date +%Y%m%d)
OUT=artifacts/scheduled_runs/$DATE

mkdir -p $OUT
agents-cli eval generate --dataset tests/eval/datasets/regression.json -o $OUT/
agents-cli eval grade --traces $OUT/eval_dataset_*.json \
    --config tests/eval/eval_config.yaml \
    --output $OUT/

#!/usr/bin/env python3
"""Exit non-zero if an eval result dips below baseline by more than --threshold."""
import argparse, json, sys

def mean(path):
    with open(path) as f:
        return json.load(f)['summary_metrics'][0]['mean_score']

p = argparse.ArgumentParser()
p.add_argument('candidate')
p.add_argument('--baseline', required=True)
p.add_argument('--threshold', type=float, default=0.05)
args = p.parse_args()

b, c = mean(args.baseline), mean(args.candidate)
drop = b - c

if drop > args.threshold:
    print(f"FAIL: mean dropped {b:.3f} → {c:.3f} "
          f"(drop {drop:.3f} > threshold {args.threshold})")
    sys.exit(1)

print(f"OK: mean {b:.3f} → {c:.3f} (drop {drop:.3f} ≤ threshold {args.threshold})")

$ python3 scripts/check_score.py \
      artifacts/grade_results/results_20261003_225158.json \
      --baseline artifacts/grade_results/results_20261003_224516.json \
      --threshold 0.1
FAIL: mean dropped 4.000 → 2.750 (drop 1.250 > threshold 0.1)
$ echo "exit: $?"
exit: 1

$ python3 scripts/check_score.py \
      artifacts/grade_results/results_20261003_224516.json \
      --baseline artifacts/grade_results/results_20261003_224516.json \
      --threshold 0.1
OK: mean 4.000 → 4.000 (drop 0.000 ≤ threshold 0.1)
$ echo "exit: $?"
exit: 0

agents-cli eval optimize \
    --dataset tests/eval/datasets/regression.json \
    --target-metric final_response_quality \
    --config tests/eval/eval_config.yaml

uvx google-agents-cli setup
agents-cli create my-agent
cd my-agent && agents-cli install

agents-cli eval generate --dataset tests/eval/datasets/basic-dataset.json \
    -o tests/eval/output/
agents-cli eval grade --traces tests/eval/output/eval_dataset_*.json \
    --config tests/eval/eval_config.yaml