Different tasks call for different metrics. Using BLEU to evaluate a customer support bot is like measuring a chef's skill by how many ingredients they used. This lesson maps each metric to the tasks it actually measures well.
# pip install rouge-score bert-score sacrebleu
import re
from rouge_score import rouge_scorer
from bert_score import score as bert_score
import sacrebleu
def normalize(text: str) -> str:
"""Lower, strip punctuation, collapse whitespace."""
text = text.lower().strip()
text = re.sub(r"[^ws]", "", text)
text = re.sub(r"s+", " ", text)
return text
def exact_match(prediction: str, reference: str) -> float:
return float(normalize(prediction) == normalize(reference))
def bleu(prediction: str, reference: str) -> float:
result = sacrebleu.sentence_bleu(prediction, [reference])
return result.score / 100 # sacrebleu returns 0–100
def rouge_l(prediction: str, reference: str) -> float:
scorer = rouge_scorer.RougeScorer(["rougeL"], use_stemmer=True)
return scorer.score(reference, prediction)["rougeL"].fmeasure
def bertscore_f1(predictions: list[str], references: list[str]) -> list[float]:
"""BERTScore requires batching — more efficient on lists."""
P, R, F1 = bert_score(predictions, references, lang="en", verbose=False)
return F1.tolist()
# Run on a test set
test_pairs = [
("The capital of France is Paris.", "Paris is the capital of France."),
("The model uses backpropagation.", "Training uses backpropagation."),
]
for pred, ref in test_pairs:
print(f"EM: {exact_match(pred, ref):.2f}")
print(f"BLEU: {bleu(pred, ref):.2f}")
print(f"ROUGE-L:{rouge_l(pred, ref):.2f}")| Task | Recommended Metrics |
|---|---|
| Q&A (factoid) | Exact Match + BERTScore (one right answer) |
| Summarization | ROUGE-L + BERTScore + human eval sample |
| RAG systems | RAGAS (faithfulness, relevancy, precision, recall) |
| Code generation | pass@k (functional correctness via unit tests) |
| Classification | Accuracy, F1, confusion matrix |
| Open-ended chat | LLM-as-judge (no reference answer exists) |
| Translation | BLEU + ChrF (handles multilingual n-grams better) |
def pass_at_k(num_correct: int, num_samples: int, k: int) -> float:
"""
Probability that at least one of k samples passes all unit tests.
Used in HumanEval, MBPP, and other code generation benchmarks.
num_correct: number of samples that passed all tests
num_samples: total samples generated (n)
k: how many samples the model is allowed to use
"""
if num_correct == 0:
return 0.0
if num_correct >= k:
return 1.0
# Combinatorial formula from Chen et al. (2021)
from math import comb
return 1.0 - comb(num_samples - num_correct, k) / comb(num_samples, k)
# Example: generate 10 solutions, 3 pass tests
# How likely is the model to have at least 1 passing solution if we pick k=3?
print(pass_at_k(num_correct=3, num_samples=10, k=3)) # → ~0.625In the next lesson, we build the most powerful tool in the eval arsenal: LLM-as-judge — using a strong model to score another model's outputs against a rubric you define.