A model cascade routes each request to the cheapest model that can handle it — and automatically escalates to a stronger model when the cheaper one isn't confident enough. A well-tuned cascade cuts API costs by 60–80% with near-zero quality loss.
Short factual queries, classification, extraction, format conversion
# Tier 1 — Fast & cheap (80% of traffic)
response = client.messages.create(
model="claude-haiku-4-5-20251001",
max_tokens=300,
messages=[{"role": "user", "content": prompt}],
)
# Cost: ~$0.0002/request Latency: ~300msimport anthropic
from pydantic import BaseModel
client = anthropic.Anthropic()
class CascadeResponse(BaseModel):
answer: str
confidence: float # 0.0 - 1.0
escalated: bool
model_used: str
def classify_query_complexity(query: str) -> str:
"""
Fast heuristic routing — before any LLM call.
Avoids even the cheapest model call for obvious cases.
"""
if len(query.split()) < 10 and "?" not in query:
return "simple"
if any(kw in query.lower() for kw in ["analyze", "compare", "explain why", "design"]):
return "complex"
return "medium"
CASCADE = [
("claude-haiku-4-5-20251001", 0.85), # Escalate if confidence < 0.85
("claude-sonnet-4-6", 0.75),
("claude-opus-4-8", 0.0), # Final tier — always trust
]
CONFIDENCE_PROMPT = """Answer the question. Then rate your confidence (0.0-1.0) that your
answer is complete and accurate. Format as JSON: {"answer": "...", "confidence": 0.XX}
Question: {question}"""
def cascade_query(question: str) -> CascadeResponse:
initial_tier = classify_query_complexity(question)
start_index = {"simple": 0, "medium": 1, "complex": 2}[initial_tier]
for model, escalation_threshold in CASCADE[start_index:]:
import json
response = client.messages.create(
model=model,
max_tokens=500,
messages=[{"role": "user", "content": CONFIDENCE_PROMPT.format(question=question)}],
)
try:
data = json.loads(response.content[0].text)
confidence = float(data.get("confidence", 0))
if confidence >= escalation_threshold or model == CASCADE[-1][0]:
return CascadeResponse(
answer=data["answer"],
confidence=confidence,
escalated=(model != CASCADE[start_index][0]),
model_used=model,
)
# else: confidence too low — escalate to next tier
except Exception:
# Parse error — escalate
continue
# Should never reach here
raise RuntimeError("Cascade exhausted without response")
result = cascade_query("What is 2 + 2?")
print(f"Model: {result.model_used}, Confidence: {result.confidence:.0%}")
# → Model: claude-haiku-4-5-20251001, Confidence: 99%
result = cascade_query("Analyze the regulatory implications of the EU AI Act for a healthcare AI startup.")
print(f"Model: {result.model_used}, Escalated: {result.escalated}")
# → Model: claude-opus-4-8, Escalated: TrueConfidence-based escalation requires extra LLM calls. A faster approach is task-based routing — classify the query type with a cheap call and route deterministically:
TASK_ROUTER_PROMPT = """Classify this query into one of: simple | medium | complex
Reply with only one word.
simple: factual lookups, format conversion, short extraction
medium: code generation, summarization, RAG Q&A
complex: multi-step analysis, ambiguous judgment, long-form research
Query: {query}"""
def route_by_task(query: str) -> str:
"""One haiku call to route — then use the right tier directly."""
response = client.messages.create(
model="claude-haiku-4-5-20251001",
max_tokens=10,
messages=[{"role": "user", "content": TASK_ROUTER_PROMPT.format(query=query)}],
)
tier = response.content[0].text.strip().lower()
return {"simple": "claude-haiku-4-5-20251001",
"medium": "claude-sonnet-4-6",
"complex": "claude-opus-4-8"}.get(tier, "claude-sonnet-4-6")In the next lesson, we add semantic caching on top of the cascade — so repeated similar queries never reach the LLM at all.