LLM 评测体系：从 MMLU 到 LMSYS Arena

import numpy as np
from scipy.special import comb

def pass_at_k(n: int, c: int, k: int) -> float:
    """
    计算 Pass@K 指标
    
    参数:
        n: 每个问题的总采样数
        c: 通过测试用例的采样数
        k: 取 K 个样本中的最优
    
    返回:
        Pass@K 概率
    """
    if n - c < k:
        return 1.0
    return 1.0 - comb(n - c, k) / comb(n, k)


# 示例：代码生成评测
# 假设每个问题采样 100 次
n_samples = 100

results = {
    "GPT-4": {"total": 164, "correct": 120},  # HumanEval
    "Claude 3": {"total": 164, "correct": 105},
    "Llama 3 70B": {"total": 164, "correct": 80},
}

print("Pass@K 指标对比（HumanEval, n=100）:")
print(f"{'Model':<15} | {'Pass@1':>8} | {'Pass@10':>8} | {'Pass@100':>8}")
print("-" * 48)

for model, data in results.items():
    # 估算每个问题的通过率
    avg_pass_rate = data["correct"] / data["total"]
    # 假设每个问题的正确数服从二项分布
    c_per_problem = n_samples * avg_pass_rate
    
    p1 = pass_at_k(n_samples, c_per_problem, 1)
    p10 = pass_at_k(n_samples, c_per_problem, 10)
    p100 = pass_at_k(n_samples, c_per_problem, 100)
    
    print(f"{model:<15} | {p1*100:>7.1f}% | {p10*100:>7.1f}% | {p100*100:>7.1f}%")

# Elo 评分系统实现
class EloRatingSystem:
    """Elo 评分系统，用于模型对战排名"""
    
    def __init__(self, k_factor: float = 32, 
                 base_rating: float = 1000):
        self.k_factor = k_factor
        self.ratings = {}
        self.base_rating = base_rating
    
    def expected_score(self, rating_a: float, 
                       rating_b: float) -> float:
        """计算 A 对 B 的期望胜率"""
        return 1.0 / (1.0 + 10 ** ((rating_b - rating_a) / 400))
    
    def update_ratings(self, model_a: str, model_b: str,
                       score_a: float):
        """
        更新两个模型的 Elo 评分
        
        score_a: 1.0 = A 胜, 0.5 = 平, 0.0 = A 负
        """
        # 确保两个模型都在系统中
        for model in [model_a, model_b]:
            if model not in self.ratings:
                self.ratings[model] = self.base_rating
        
        # 计算期望分数
        expected_a = self.expected_score(
            self.ratings[model_a], self.ratings[model_b]
        )
        expected_b = 1 - expected_a
        
        # 更新评分
        self.ratings[model_a] += self.k_factor * (
            score_a - expected_a
        )
        self.ratings[model_b] += self.k_factor * (
            (1 - score_a) - expected_b
        )
    
    def get_rankings(self) -> list:
        """返回排序后的排行榜"""
        sorted_models = sorted(
            self.ratings.items(),
            key=lambda x: x[1],
            reverse=True
        )
        return sorted_models


# 模拟对战
elo = EloRatingSystem(k_factor=32)
import random

models = ["Model A", "Model B", "Model C", "Model D"]
# 假设真实实力：A > B > C > D
true_skills = {"A": 0.8, "B": 0.6, "C": 0.4, "D": 0.2}

# 模拟 500 场对战
for _ in range(500):
    a, b = random.sample(models, 2)
    skill_diff = true_skills[a] - true_skills[b]
    # 实力强的获胜概率更高（加入随机性）
    win_prob = 0.5 + 0.3 * skill_diff
    score = 1.0 if random.random() < win_prob else 0.0
    elo.update_ratings(f"Model {a}", f"Model {b}", score)

print("Elo 排行榜（500 场模拟对战）:")
for rank, (model, rating) in enumerate(elo.get_rankings(), 1):
    print(f"  #{rank}: {model:<10} Elo={rating:.0f}")

from dataclasses import dataclass
from typing import Callable
import json

@dataclass
class EvalSample:
    """单个评测样本"""
    id: str
    input: str          # 用户输入
    reference: str      # 参考答案（可选）
    rubric: str         # 评分标准
    category: str       # 类别标签
    difficulty: str     # 难度等级


@dataclass
class EvalResult:
    """评测结果"""
    sample_id: str
    score: float        # 0-100
    category: str
    details: dict       # 详细评分信息


class CustomEvalFramework:
    """自定义 LLM 评测框架"""
    
    def __init__(self, judge_llm: str = "gpt-4"):
        self.judge_llm = judge_llm
        self.samples = []
        self.results = []
    
    def add_sample(self, sample: EvalSample):
        self.samples.append(sample)
    
    def evaluate_sample(self, model_output: str, 
                       sample: EvalSample) -> EvalResult:
        """
        使用 LLM-as-Judge 评估单个样本
        使用结构化 Prompt 减少评估偏差
        """
        judge_prompt = f"""你是一位公正的评测专家。请根据以下标准评估模型的输出。

【用户输入】
{sample.input}

【模型输出】
{model_output}

【评分标准】
{sample.rubric}

请从以下维度评分（0-100）：
1. 准确性：信息是否正确，有无幻觉
2. 完整性：是否覆盖了关键要点
3. 相关性：是否回答了用户的问题
4. 清晰度：表达是否清晰、结构化

输出格式（严格 JSON）：
{{"accuracy": 0-100, "completeness": 0-100, 
  "relevance": 0-100, "clarity": 0-100,
  "overall": 0-100, "reason": "简要评价"}}"""
        
        # response = call_llm(judge_prompt)
        # scores = json.loads(response)
        # 此处仅为框架示意
        scores = {
            "accuracy": 85, "completeness": 78,
            "relevance": 92, "clarity": 80,
            "overall": 84,
            "reason": "回答准确且相关，但缺少部分细节"
        }
        
        return EvalResult(
            sample_id=sample.id,
            score=scores["overall"],
            category=sample.category,
            details=scores
        )
    
    def generate_report(self) -> dict:
        """生成综合评测报告"""
        categories = {}
        for r in self.results:
            if r.category not in categories:
                categories[r.category] = []
            categories[r.category].append(r.score)
        
        report = {
            "overall_score": np.mean([r.score for r in self.results]),
            "category_scores": {
                cat: {
                    "mean": np.mean(scores),
                    "std": np.std(scores),
                    "min": min(scores),
                    "max": max(scores),
                    "count": len(scores)
                }
                for cat, scores in categories.items()
            }
        }
        return report


# 使用示例
framework = CustomEvalFramework()

# 添加评测样本
framework.add_sample(EvalSample(
    id="qa-001",
    input="请解释量子纠缠的概念",
    reference="量子纠缠是量子力学现象...",
    rubric="准确解释量子纠缠，包含定义、特征、应用",
    category="知识理解",
    difficulty="中等"
))
print("评测框架初始化完成")