NLP 评估指标：BLEU, ROUGE, METEOR

import numpy as np

# 人工评估 vs 自动评估的对比模拟
print("=== NLP 评估方法对比 ===\n")

manual_eval = {
    "cost_per_sample": "$0.50 - $2.00",
    "throughput": "~500 samples/hour (per annotator)",
    "reliability": "high (but subjective)",
    "inter_annotator_agreement": "usually 0.6-0.8 (Kappa)",
}

auto_eval = {
    "cost_per_sample": "$0.001 (compute cost)",
    "throughput": "~100,000 samples/second",
    "reliability": "medium (depends on metric)",
    "correlation_with_human": "BLEU: 0.3-0.6, BERTScore: 0.5-0.8",
}

print("Human Evaluation:")
for k, v in manual_eval.items():
    print(f"  {k}: {v}")

print("\nAutomatic Evaluation:")
for k, v in auto_eval.items():
    print(f"  {k}: {v}")

# Why multiple metrics?
print("\n=== Why Multiple Metrics? ===")
print("No single metric fully captures NLP quality:")
print("  - BLEU fits MT, not dialogue generation")
print("  - ROUGE fits summarization, not translation")
print("  - Perplexity fits LM, not generation quality")
print("  - BERTScore is general but expensive")
print("Best practice: use 2-3 complementary metrics")

import numpy as np

# n-gram overlap: the core concept of NLP auto-evaluation
def get_ngrams(text, n):
    """Extract n-grams from text"""
    tokens = text.lower().split()
    return [tuple(tokens[i:i+n]) for i in range(len(tokens) - n + 1)]

def ngram_overlap(reference, hypothesis, n):
    """Calculate n-gram overlap ratio"""
    ref_ngrams = get_ngrams(reference, n)
    hyp_ngrams = get_ngrams(hypothesis, n)

    if not hyp_ngrams:
        return 0.0

    ref_counts = {}
    for ng in ref_ngrams:
        ref_counts[ng] = ref_counts.get(ng, 0) + 1

    matched = 0
    for ng in hyp_ngrams:
        if ref_counts.get(ng, 0) > 0:
            matched += 1
            ref_counts[ng] -= 1

    return matched / len(hyp_ngrams)

# Demo
ref = "the cat sat on the mat"
hyp1 = "the cat is sitting on the mat"
hyp2 = "the dog played in the yard"

print("=== n-gram Overlap Demo ===")
for n in [1, 2, 3]:
    s1 = ngram_overlap(ref, hyp1, n)
    s2 = ngram_overlap(ref, hyp2, n)
    print(f"  {n}-gram: good_trans={s1:.3f}, bad_trans={s2:.3f}")

# Higher n = stricter match, lower score
# BLEU combines multiple n-gram scores

import numpy as np
from collections import Counter

def get_ngrams(tokens, n):
    """Extract n-gram list"""
    return [tuple(tokens[i:i+n]) for i in range(len(tokens) - n + 1)]

def modified_precision(reference, hypothesis, n):
    """Modified n-gram precision (BLEU core)"""
    ref_ngrams = Counter(get_ngrams(reference, n))
    hyp_ngrams = Counter(get_ngrams(hypothesis, n))

    # Clip: count in hyp cannot exceed count in ref
    clipped_count = sum(
        min(hyp_count, ref_ngrams.get(ng, 0))
        for ng, hyp_count in hyp_ngrams.items()
    )
    total_count = sum(hyp_ngrams.values())

    return clipped_count / total_count if total_count > 0 else 0

def compute_bleu(reference, hypothesis, max_n=4):
    """Calculate BLEU score (simplified)"""
    ref_tokens = reference.split()
    hyp_tokens = hypothesis.split()

    precisions = []
    for n in range(1, max_n + 1):
        p = modified_precision(ref_tokens, hyp_tokens, n)
        precisions.append(p)
        print(f"  {n}-gram precision: {p:.4f}")

    # Geometric mean
    log_avg = np.mean([np.log(max(p, 1e-10)) for p in precisions])
    geom_mean = np.exp(log_avg)

    # Brevity penalty
    ref_len = len(ref_tokens)
    hyp_len = len(hyp_tokens)
    bp = np.exp(1 - ref_len / hyp_len) if hyp_len < ref_len else 1.0

    bleu = bp * geom_mean
    print(f"  BP = {bp:.4f}, Geometric Mean = {geom_mean:.4f}")
    return bleu

ref = "the cat sat on the mat"
print("=== BLEU Calculation Demo ===")
print("\nHypothesis 1: 'the cat is sitting on the mat'")
s1 = compute_bleu(ref, "the cat is sitting on the mat")
print(f"  BLEU = {s1:.4f}\n")

print("Hypothesis 2: 'the dog played in the yard'")
s2 = compute_bleu(ref, "the dog played in the yard")
print(f"  BLEU = {s2:.4f}")

# BLEU score interpretation
import numpy as np

print("=== BLEU Score Interpretation ===\n")

bleu_guide = [
    (0, 10, "Unusable", "Translation barely understandable"),
    (10, 20, "Understandable", "Can guess meaning, many errors"),
    (20, 30, "Good", "Mostly fluent, some errors"),
    (30, 40, "Very Good", "High quality, minor issues"),
    (40, 50, "Excellent", "Near professional level"),
    (50, 100, "Human Level", "Indistinguishable from human"),
]

print("BLEU Score Reference:")
for low, high, level, desc in bleu_guide:
    print(f"  {low:3d}-{high:3d}: {level:<15} - {desc}")

# Multi-reference BLEU
print("\n=== Multi-reference BLEU ===")
print("In practice, each source sentence has multiple reference")
print("translations. BLEU takes the best match across references.")
print()
print("Example:")
print("  Source: 'The cat sat on the mat'")
print("  Ref 1:  'Le chat etait assis sur le tapis'")
print("  Ref 2:  'Le chat s'est assis sur le paillasson'")
print("  Hyp:    'Le chat etait sur le tapis'")
print("  -> Ref 2 gives better match for 'sur le'")
print("  -> Multi-ref BLEU is fairer than single-ref")

from collections import Counter

def rouge_n(reference, hypothesis, n):
    """Calculate ROUGE-N recall, precision and F1"""
    ref_tokens = reference.split()
    hyp_tokens = hypothesis.split()

    ref_ngrams = Counter(
        tuple(ref_tokens[i:i+n]) for i in range(len(ref_tokens) - n + 1)
    )
    hyp_ngrams = Counter(
        tuple(hyp_tokens[i:i+n]) for i in range(len(hyp_tokens) - n + 1)
    )

    # Overlap count (take min)
    overlap = sum(
        min(ref_ngrams[ng], hyp_ngrams[ng]) for ng in hyp_ngrams
    )

    ref_total = sum(ref_ngrams.values())
    recall = overlap / ref_total if ref_total > 0 else 0

    hyp_total = sum(hyp_ngrams.values())
    precision = overlap / hyp_total if hyp_total > 0 else 0

    f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0

    return recall, precision, f1

# Demo
ref = "the cat sat on the mat and the dog played in the yard"
hyp = "the cat was sitting on the mat the dog was playing outside"

print("=== ROUGE-N Calculation ===")
for n in [1, 2, 3]:
    r, p, f1 = rouge_n(ref, hyp, n)
    print(f"  ROUGE-{n}: R={r:.4f}, P={p:.4f}, F1={f1:.4f}")

def lcs_length(X, Y):
    """Calculate Longest Common Subsequence length"""
    m, n = len(X), len(Y)
    dp = [[0] * (n + 1) for _ in range(m + 1)]

    for i in range(1, m + 1):
        for j in range(1, n + 1):
            if X[i-1] == Y[j-1]:
                dp[i][j] = dp[i-1][j-1] + 1
            else:
                dp[i][j] = max(dp[i-1][j], dp[i][j-1])

    return dp[m][n]

def rouge_l(reference, hypothesis):
    """Calculate ROUGE-L based on LCS"""
    ref_tokens = reference.split()
    hyp_tokens = hypothesis.split()

    lcs_len = lcs_length(ref_tokens, hyp_tokens)

    recall = lcs_len / len(ref_tokens) if ref_tokens else 0
    precision = lcs_len / len(hyp_tokens) if hyp_tokens else 0
    f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0

    return recall, precision, f1, lcs_len

# Demo: ROUGE-L advantage
ref = "the cat sat on the mat"
hyp = "the cat is sitting on the mat"

print("=== ROUGE-L vs ROUGE-N ===")
print(f"Ref: '{ref}'")
print(f"Hyp: '{hyp}'\n")

rl_r, rl_p, rl_f1, lcs_len = rouge_l(ref, hyp)
print(f"ROUGE-L: LCS_len={lcs_len}, R={rl_r:.4f}, P={rl_p:.4f}, F1={rl_f1:.4f}")

r2_r, r2_p, r2_f1 = rouge_n(ref, hyp, 2)
print(f"ROUGE-2: R={r2_r:.4f}, P={r2_p:.4f}, F1={r2_f1:.4f}")

print("\nROUGE-L captures 'the cat _ _ on the mat' as LCS (5 words)")
print("Even with insertions, ROUGE-L still finds core content coverage")

from collections import Counter
import numpy as np

# Mini synonym dictionary (real METEOR uses WordNet)
SYNONYMS = {
    "big": {"large", "huge", "enormous"},
    "small": {"little", "tiny", "mini"},
    "good": {"great", "excellent", "fine"},
    "cat": {"feline", "kitty", "kitten"},
    "dog": {"canine", "puppy", "pooch"},
    "happy": {"glad", "joyful", "cheerful"},
}

def meteor_match(ref_word, hyp_word):
    """METEOR match: exact > stem > synonym"""
    if ref_word == hyp_word:
        return "exact"

    # Simple stem match
    ref_stem = ref_word.rstrip("ingsed")
    hyp_stem = hyp_word.rstrip("ingsed")
    if ref_stem == hyp_stem and len(ref_stem) > 2:
        return "stem"

    # Synonym match
    syn_ref = SYNONYMS.get(ref_word, set())
    syn_hyp = SYNONYMS.get(hyp_word, set())
    if hyp_word in syn_ref or ref_word in syn_hyp:
        return "synonym"

    return None

def compute_meteor(reference, hypothesis):
    """Simplified METEOR calculation"""
    ref_tokens = reference.lower().split()
    hyp_tokens = hypothesis.lower().split()

    # Greedy alignment
    matched_ref = set()
    matched_hyp = set()

    for i, hyp_w in enumerate(hyp_tokens):
        for j, ref_w in enumerate(ref_tokens):
            if j in matched_ref or i in matched_hyp:
                continue
            if meteor_match(ref_w, hyp_w):
                matched_ref.add(j)
                matched_hyp.add(i)
                break

    precision = len(matched_hyp) / len(hyp_tokens) if hyp_tokens else 0
    recall = len(matched_ref) / len(ref_tokens) if ref_tokens else 0

    # F-mean with alpha=9 (recall weighted 9x)
    alpha = 9
    f_mean = (1 + alpha**3) * precision * recall / \
             (alpha**3 * precision + recall) if (precision + recall) > 0 else 0

    # Chunk penalty
    chunks = 1
    sorted_hyp = sorted(matched_hyp)
    for i in range(1, len(sorted_hyp)):
        if sorted_hyp[i] != sorted_hyp[i-1] + 1:
            chunks += 1

    penalty = 0.5 * (chunks / max(len(matched_hyp), 1)) ** 3
    score = f_mean * (1 - penalty)

    return score, precision, recall, chunks

ref = "the big cat sat on the mat"
hyp1 = "the large feline sat on the mat"
hyp2 = "the big mat on sat the cat"

print("=== METEOR Calculation ===")
for hyp, label in [(hyp1, "Synonym"), (hyp2, "Word Reorder")]:
    score, p, r, ch = compute_meteor(ref, hyp)
    print(f"  {label}: METEOR={score:.3f}, P={p:.3f}, R={r:.3f}, chunks={ch}")

# BLEU vs METEOR comparison
import numpy as np

def simple_bleu(reference, hypothesis, max_n=4):
    """Simplified BLEU"""
    from collections import Counter
    ref_tokens = reference.lower().split()
    hyp_tokens = hypothesis.lower().split()

    precisions = []
    for n in range(1, max_n + 1):
        ref_ng = Counter(
            tuple(ref_tokens[i:i+n]) for i in range(len(ref_tokens)-n+1)
        )
        hyp_ng = Counter(
            tuple(hyp_tokens[i:i+n]) for i in range(len(hyp_tokens)-n+1)
        )
        clipped = sum(
            min(c, ref_ng.get(ng, 0)) for ng, c in hyp_ng.items()
        )
        total = sum(hyp_ng.values())
        precisions.append(clipped / total if total > 0 else 0)

    log_avg = np.mean([np.log(max(p, 1e-10)) for p in precisions])
    rl, hl = len(ref_tokens), len(hyp_tokens)
    bp = np.exp(1 - rl/hl) if hl < rl else 1.0
    return bp * np.exp(log_avg)

print("=== BLEU vs METEOR ===\n")

test_cases = [
    ("the cat sat on the mat", "the cat sat on the mat", "Exact match"),
    ("the cat sat on the mat", "the feline sat on the mat", "Synonym"),
    ("the cat sat on the mat", "the cat was sitting on mat", "Stem variant"),
    ("the cat sat on the mat", "on the mat sat the cat", "Reorder"),
    ("the cat sat on the mat", "the dog played in the yard", "Different"),
]

for ref, hyp, desc in test_cases:
    bleu = simple_bleu(ref, hyp)
    met, p, r, ch = compute_meteor(ref, hyp)
    arrow = "  ^ METEOR higher (more tolerant)" if met > bleu else "  BLEU higher or equal"
    print(f"{desc}:")
    print(f"  BLEU={bleu:.4f}, METEOR={met:.4f}")
    print(f"{arrow}\n")

import numpy as np

def compute_perplexity(model_probs, smooth=1e-10):
    """Calculate perplexity from model probabilities
    model_probs: list of probabilities assigned to correct words
    """
    N = len(model_probs)
    if N == 0:
        return float('inf')

    # Cross-entropy (natural log)
    log_probs = [np.log(max(p, smooth)) for p in model_probs]
    cross_entropy = -np.mean(log_probs)

    # Perplexity = exp(cross_entropy)
    perplexity = np.exp(cross_entropy)
    return perplexity

sentence = ["the", "cat", "sat", "on", "the", "mat"]

# Model A: n-gram (moderate)
probs_a = [0.3, 0.4, 0.2, 0.5, 0.8, 0.6]
pp_a = compute_perplexity(probs_a)

# Model B: random (uniform over 10000 vocab)
probs_b = [1/10000] * 6
pp_b = compute_perplexity(probs_b)

# Model C: Transformer (excellent)
probs_c = [0.7, 0.8, 0.6, 0.9, 0.95, 0.85]
pp_c = compute_perplexity(probs_c)

print("=== Perplexity Calculation ===")
print(f"  Model A (n-gram):    PP = {pp_a:.2f}")
print(f"  Model B (random):    PP = {pp_b:.2f}")
print(f"  Model C (Transform): PP = {pp_c:.2f}")

print(f"\nInterpretation:")
print(f"  PP={pp_b:.0f}: like choosing from {pp_b:.0f} words randomly")
print(f"  PP={pp_a:.1f}: like choosing from {pp_a:.1f} words")
print(f"  PP={pp_c:.1f}: like choosing from {pp_c:.1f} words")

import numpy as np

print("=== Perplexity vs Cross-Entropy ===\n")

# PP = 2^H (base 2) or exp(H) (base e)
# Each 1-bit reduction in cross-entropy halves perplexity

print(f"{'Cross-Ent H':<14} {'PP (base 2)':<14} {'PP (base e)':<14} {'Quality'}")
print("-" * 55)

for h in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]:
    pp2 = 2 ** h
    ppe = np.exp(h)
    if h <= 4:
        desc = "Good"
    elif h <= 7:
        desc = "Fair"
    else:
        desc = "Poor"
    print(f"{h:<14} {pp2:<14.1f} {ppe:<14.1f} {desc}")

print("\nKey relationship:")
print("  PP = 2^H  <=>  H = log2(PP)")
print("  Each 1-bit CE reduction halves PP")
print("  CE 7->6: PP 128->64 (big improvement)")
print("  CE 4->3: PP 16->8 (same improvement)")

print("\n=== Perplexity Limitations ===")
print("1. Only comparable on SAME test set")
print("2. Cannot compare across datasets")
print("3. Low PP != good generation (may overfit)")
print("4. PP only evaluates probability, not coherence")
print("5. Sensitive to vocabulary size")

import numpy as np

def cosine_sim(a, b):
    """Cosine similarity"""
    return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b) + 1e-10)

def greedy_match(ref_embeds, hyp_embeds):
    """Greedy matching: each token finds best match"""
    ref_to_hyp = []
    for ref_emb in ref_embeds:
        sims = [cosine_sim(ref_emb, h) for h in hyp_embeds]
        ref_to_hyp.append(max(sims))

    hyp_to_ref = []
    for hyp_emb in hyp_embeds:
        sims = [cosine_sim(r, hyp_emb) for r in ref_embeds]
        hyp_to_ref.append(max(sims))

    return np.mean(hyp_to_ref), np.mean(ref_to_hyp)

def bert_score_simulated(reference, hypothesis):
    """Simulated BERTScore computation"""
    np.random.seed(42)

    ref_words = reference.split()
    hyp_words = hypothesis.split()

    # Simulated BERT embeddings (semantically similar = closer vectors)
    emb = {}
    base_cat = np.random.randn(768)
    emb["cat"] = base_cat
    emb["feline"] = base_cat * 0.85 + np.random.randn(768) * 0.15
    emb["kitten"] = base_cat * 0.75 + np.random.randn(768) * 0.2
    emb["dog"] = np.random.randn(768)
    emb["sat"] = np.random.randn(768)
    emb["sitting"] = emb["sat"] * 0.8 + np.random.randn(768) * 0.2
    emb["the"] = np.random.randn(768)
    emb["on"] = np.random.randn(768)
    emb["mat"] = np.random.randn(768)

    ref_e = [emb.get(w, np.random.randn(768)) for w in ref_words]
    hyp_e = [emb.get(w, np.random.randn(768)) for w in hyp_words]

    precision, recall = greedy_match(ref_e, hyp_e)
    f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0

    return precision, recall, f1

ref = "the cat sat on the mat"
hyp1 = "the feline was sitting on the mat"
hyp2 = "the dog played in the yard"

print("=== BERTScore (Simulated) ===")
for hyp, label in [(hyp1, "Synonym replace"), (hyp2, "Completely different")]:
    p, r, f1 = bert_score_simulated(ref, hyp)
    print(f"  {label}: P={p:.4f}, R={r:.4f}, F1={f1:.4f}")

# Real BERTScore usage (requires bert-score library)
print("=== Real BERTScore Usage ===\n")

print("# Install:")
print("  pip install bert-score\n")

print("# Basic usage:")
print("from bert_score import score")
print()
print("cands = ['the cat is sitting on the mat']")
print("refs = ['the cat sat on the mat']")
print("P, R, F1 = score(cands, refs, lang='en', verbose=True)")
print("print(f'P={P.mean():.4f}, R={R.mean():.4f}, F1={F1.mean():.4f}')")
print()

print("# Batch evaluation:")
print("cands = [")
print("    'the cat is sitting on the mat',")
print("    'the dog played in the yard',")
print("]")
print("refs = ['the cat sat on the mat'] * 2")
print("P, R, F1 = score(cands, refs, lang='en')")
print()

print("# Multi-language support:")
print("score(cands, refs, lang='zh')  # Chinese")
print("score(cands, refs, lang='fr')  # French")
print()

print("=== BLEU vs BERTScore ===")
print("BLEU:")
print("  + Fast (no GPU needed)")
print("  + Simple implementation")
print("  - No semantic understanding")
print("  - Sensitive to word order")
print("  - Lower human correlation")
print()
print("BERTScore:")
print("  + Captures semantic equivalence")
print("  + High human correlation")
print("  + Multi-language native support")
print("  - Needs GPU and pretrained model")
print("  - Expensive (~50ms/sentence on GPU)")

# sacrebleu practical usage
print("=== sacrebleu Practical Guide ===\n")

print("# Install: pip install sacrebleu")
print()
print("import sacrebleu")
print()

print("# 1. Basic BLEU calculation:")
print("refs = ['the cat sat on the mat', 'a cat is sitting on the mat']")
print("hyps = ['the cat sat on the mat']")
print("bleu = sacrebleu.corpus_bleu(hyps, [refs])")
print("print(bleu.score)  # Output: 100.0")
print()

print("# 2. Signed output (required for papers):")
print("print(bleu.format())")
print("# BLEU = 45.23 67.8/48.9/38.2/30.1 (BP = 1.000 ratio = 1.05)")
print()

print("# 3. Built-in test sets:")
print("# Command line: sacrebleu -t wmt17 -l zh-en < hyp.txt")
print()

print("# 4. chrF: character-level F-score")
print("# Better for morphologically rich languages")
print("chrf = sacrebleu.corpus_chrf(hyps, [refs])")
print()

print("# 5. TER: Translation Edit Rate")
print("# Lower is better (measures edit distance)")
print("ter = sacrebleu.corpus_ter(hyps, [refs])")

# Hugging Face evaluate library
print("=== evaluate Library ===\n")

print("from evaluate import load")
print()

print("# 1. Load metrics:")
print("bleu = load('bleu')")
print("rouge = load('rouge')")
print("meteor = load('meteor')")
print("bertscore = load('bertscore')")
print()

print("# 2. Unified API:")
print("predictions = ['the cat sat on the mat']")
print("references = ['the cat sat on the mat']")
print()

print("bleu_result = bleu.compute(")
print("    predictions=predictions,")
print("    references=references")
print(")")
print("print(bleu_result)  # {'bleu': 100.0, ...}")
print()

print("# 3. Batch evaluation:")
print("for name in ['bleu', 'rouge', 'meteor']:")
print("    metric = load(name)")
print("    score = metric.compute(")
print("        predictions=test_preds,")
print("        references=test_refs")
print("    )")
print("    results[name] = score")
print()

print("# 4. Model comparison:")
print("models = ['gpt2', 'gpt2-medium', 'gpt2-large']")
print("for model_name in models:")
print("    bleu = load('bleu')")
print("    score = bleu.compute(predictions=preds, references=refs)")
print("    print(f'{model_name}: BLEU={score["bleu"]:.2f}')")