A two-word change to your system prompt can silently drop task completion from 92% to 67%. Prompt regression testing catches these silent degradations before users do — by running your full eval suite on every prompt change, just like unit tests run on every code change.
Unlike code bugs that crash with a stack trace, prompt regressions are silent. The model still returns a response. It still looks reasonable. But completion rate, accuracy, or tone quietly worsen. Without a regression test suite, you discover the regression from a spike in user complaints.
A team changed “You are a helpful assistant” to “You are a concise assistant.” Answer quality on open-ended questions dropped 18% (users found answers too terse), but answer quality on factoid questions improved 12%. Neither change was caught until an A/B test ran for 2 weeks.
Explore each version to see what changed and how the eval scores shifted:
Added numbered-steps instruction. +3% improvement. Zero regressions from v2.
90% pass rate on golden eval setPROMPTS["support_v2_1"] = {
"system": """You are a customer support specialist for TechCorp.
Your goal is to resolve issues quickly and empathetically.
Always acknowledge the customer's frustration before offering solutions.
Be thorough — incomplete answers require follow-up and waste time.
Format your response with numbered steps when providing instructions.""",
"version": "2.1.0",
}
# compare_prompts("v2", "v2_1") output:
# {
# "old_avg": 0.87, "new_avg": 0.90,
# "delta": +0.03,
# "regressions": [], # Zero individual case regressions
# "approved": True # CI gate passes → safe to deploy
# }# prompts/registry.py — centralize all prompts + versions
PROMPTS = {
"support_v1": {
"system": "You are a helpful customer support assistant for TechCorp. Be concise.",
"version": "1.0.0",
"created": "2026-01-01",
},
"support_v2": {
"system": """You are a customer support specialist for TechCorp.
Your goal is to resolve issues quickly and empathetically.
Always acknowledge the customer's frustration before offering solutions.
Be thorough — incomplete answers require follow-up and waste time.""",
"version": "2.0.0",
"created": "2026-03-15",
},
"support_v2_1": {
"system": """You are a customer support specialist for TechCorp.
Your goal is to resolve issues quickly and empathetically.
Always acknowledge the customer's frustration before offering solutions.
Be thorough — incomplete answers require follow-up and waste time.
Format your response with numbered steps when providing instructions.""",
"version": "2.1.0",
"created": "2026-05-01",
},
}
ACTIVE_PROMPT = PROMPTS["support_v2_1"]import anthropic
import json
from pathlib import Path
from dataclasses import dataclass
client = anthropic.Anthropic()
@dataclass
class EvalResult:
question: str
response: str
score: float
passed: bool
def run_eval(prompt_key: str, eval_set_path: str, threshold: float = 0.8) -> list[EvalResult]:
"""Run an eval set against a specific prompt version."""
prompt = PROMPTS[prompt_key]
eval_cases = json.loads(Path(eval_set_path).read_text())
results = []
for case in eval_cases:
response = client.messages.create(
model="claude-haiku-4-5-20251001",
max_tokens=500,
system=prompt["system"],
messages=[{"role": "user", "content": case["question"]}],
)
answer = response.content[0].text
# Judge with a separate call
judge = client.messages.create(
model="claude-sonnet-4-6",
max_tokens=100,
messages=[{
"role": "user",
"content": f"""Rate this support response 1-10.
Customer question: {case['question']}
Expected behavior: {case['expected_behavior']}
Actual response: {answer}
Output only the number.""",
}],
)
score = float(judge.content[0].text.strip()) / 10
results.append(EvalResult(
question=case["question"],
response=answer,
score=score,
passed=score >= threshold,
))
return results
def compare_prompts(old_key: str, new_key: str, eval_path: str) -> dict:
"""A/B comparison: old prompt vs new prompt on same eval set."""
old_results = run_eval(old_key, eval_path)
new_results = run_eval(new_key, eval_path)
old_avg = sum(r.score for r in old_results) / len(old_results)
new_avg = sum(r.score for r in new_results) / len(new_results)
regressions = [
(old.question, old.score, new.score)
for old, new in zip(old_results, new_results)
if new.score < old.score - 0.1 # 10% drop on any individual case
]
return {
"old_avg": old_avg,
"new_avg": new_avg,
"delta": new_avg - old_avg,
"regressions": regressions,
"approved": new_avg >= old_avg and len(regressions) == 0,
}
# In CI:
result = compare_prompts("support_v2", "support_v2_1", "eval/support_golden.json")
if not result["approved"]:
print(f"BLOCKED: New prompt regressed {len(result['regressions'])} cases")
exit(1)# .github/workflows/prompt-eval.yml
name: Prompt Regression Check
on:
pull_request:
paths:
- 'prompts/**' # Only runs when prompts change
- 'eval/**'
jobs:
regression-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Run prompt regression eval
run: python eval/regression_check.py
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
OLD_PROMPT_KEY: support_v2
NEW_PROMPT_KEY: support_v2_1The eval set is the foundation. Invest in it once, benefit forever:
In the next lesson, we go on offense — adversarial testing and red-teaming techniques that find the inputs that break your AI system before attackers do.