The quality of your training data determines the ceiling of your fine-tuned model. You can have perfect hyperparameters and the best hardware — noisy data will produce a noisy model. Garbage in, garbage out applies more strictly to fine-tuning than anywhere else in ML.
Use when:
Simple instruction → response tasks, no conversation history needed
{
"instruction": "Extract the price from this product description.",
"input": "The XR-500 headphones are available for $149.99 on Amazon.",
"output": "$149.99"
}
# ── Formatting into model input ──────────────────────────────
def format_alpaca(example: dict) -> str:
if example.get("input"):
return (
f"### Instruction:\n{example['instruction']}\n\n"
f"### Input:\n{example['input']}\n\n"
f"### Response:\n{example['output']}"
)
return (
f"### Instruction:\n{example['instruction']}\n\n"
f"### Response:\n{example['output']}"
)import anthropic
import json
from pathlib import Path
client = anthropic.Anthropic()
SEED_EXAMPLES = [
{"instruction": "Classify this email as spam or not spam.", "input": "Congratulations! You've won...", "output": "spam"},
{"instruction": "Classify this email as spam or not spam.", "input": "Hi, your invoice is attached.", "output": "not spam"},
]
GENERATION_PROMPT = """Generate {n} diverse training examples for this task: email spam classification.
Each example must be a JSON object with:
- "instruction": exactly "Classify this email as spam or not spam."
- "input": a realistic email body (50–200 words)
- "output": exactly "spam" or "not spam"
Requirements:
- Vary writing style, sender personas, and topics
- Include ambiguous cases (newsletters, receipts, cold outreach)
- Output a JSON array of {n} objects only — no other text
Seed examples for style reference:
{seeds}"""
def generate_examples(n: int = 50) -> list[dict]:
response = client.messages.create(
model="claude-sonnet-4-6",
max_tokens=4000,
messages=[{
"role": "user",
"content": GENERATION_PROMPT.format(
n=n,
seeds=json.dumps(SEED_EXAMPLES[:2], indent=2),
),
}],
)
return json.loads(response.content[0].text)
# Generate in batches
all_examples = []
for batch in range(20): # 20 × 50 = 1,000 examples
batch_examples = generate_examples(50)
all_examples.extend(batch_examples)
print(f"Generated {len(all_examples)} examples so far...")
# Save
Path("data/train.json").write_text(json.dumps(all_examples, indent=2))from datasketch import MinHash, MinHashLSH
def deduplicate(examples: list[dict], threshold: float = 0.8) -> list[dict]:
"""Remove near-duplicate examples using MinHash LSH."""
lsh = MinHashLSH(threshold=threshold, num_perm=128)
unique = []
for i, ex in enumerate(examples):
m = MinHash(num_perm=128)
text = f"{ex['instruction']} {ex.get('input', '')} {ex['output']}"
for word in text.lower().split():
m.update(word.encode())
if not lsh.query(m): # No similar doc found
lsh.insert(str(i), m)
unique.append(ex)
print(f"Dedup: {len(examples)} → {len(unique)} examples")
return unique
def quality_filter(examples: list[dict]) -> list[dict]:
"""Remove examples that are too short, too long, or malformed."""
filtered = []
for ex in examples:
output = ex.get("output", "")
if len(output) < 5:
continue # Too short
if len(output) > 2000:
continue # Probably truncated or wrong
if "{{" in output or "TODO" in output:
continue # Template artifacts
filtered.append(ex)
return filteredSample 50 examples for human review before training. Errors that look subtle in data preview compound dramatically in the model. An hour of manual review catches issues that would take days to debug post-training.
With quality data in hand, the next lesson is a full end-to-end training walkthrough — from dataset loading through model checkpoint, with wandb loss tracking.