生成模型评估：FID, IS, CLIP Score

import torch
import torch.nn.functional as F
from torchvision import models

# 加载预训练 Inception 网络作为特征提取器
inception = models.inception_v3(weights=models.Inception_V3_Weights.DEFAULT)
inception.fc = torch.nn.Identity()  # 移除分类头
inception.eval()

def extract_features(images: torch.Tensor) -> torch.Tensor:
    """提取 2048 维 Inception 特征"""
    with torch.no_grad():
        features = inception(images)
    return features  # shape: (N, 2048)

import numpy as np
from scipy import linalg

def compute_statistics(features: np.ndarray):
    """计算特征的均值向量和协方差矩阵"""
    mu = np.mean(features, axis=0)
    sigma = np.cov(features, rowvar=False)
    return mu, sigma

# 使用示例
real_features = extract_features(real_images).numpy()
fake_features = extract_features(fake_images).numpy()
mu_r, sigma_r = compute_statistics(real_features)
mu_f, sigma_f = compute_statistics(fake_features)
print(f"Real mu shape: {mu_r.shape}, Sigma shape: {sigma_r.shape}")

import torch
import torch.nn.functional as F
import numpy as np

def compute_inception_score(preds: np.ndarray, splits: int = 10):
    """
    计算 Inception Score
    preds: (N, 1000) Inception 网络预测概率
    """
    scores = []
    N = preds.shape[0]
    split_size = N // splits

    for i in range(splits):
        part = preds[i * split_size : (i + 1) * split_size]
        # KL 散度: KL(p(y|x) || p(y))
        marginal = np.mean(part, axis=0)  # p(y)
        kl = part * (np.log(part) - np.log(marginal))  # KL per sample
        kl = np.sum(kl, axis=1)  # sum over classes
        scores.append(np.exp(np.mean(kl)))  # exp(E[KL])

    return np.mean(scores), np.std(scores)

print(f"IS = {mean_is:.2f} +/- {std_is:.2f}")

import torch
from torch.utils.data import DataLoader
from torchvision import transforms

# 完整的 IS 计算流程
@torch.no_grad()
def get_inception_predictions(
    generator: torch.nn.Module,
    inception: torch.nn.Module,
    num_samples: int = 50000,
    batch_size: int = 100,
    device: str = "cuda"
):
    """从生成器采样并通过 Inception 获取预测"""
    all_preds = []
    num_batches = num_samples // batch_size

    for _ in range(num_batches):
        noise = torch.randn(batch_size, 100, device=device)
        fake_images = generator(noise)
        fake_images = F.interpolate(fake_images, size=(299, 299), mode="bilinear")
        preds = torch.softmax(inception(fake_images), dim=1)
        all_preds.append(preds.cpu().numpy())

    return np.concatenate(all_preds, axis=0)

preds = get_inception_predictions(G, inception, num_samples=50000)
mean_is, std_is = compute_inception_score(preds)
print(f"Inception Score: {mean_is:.2f} +/- {std_is:.2f}")

import numpy as np
from scipy import linalg

def calculate_fid(
    mu1: np.ndarray, sigma1: np.ndarray,
    mu2: np.ndarray, sigma2: np.ndarray
) -> float:
    """
    计算 Fréchet Inception Distance
    mu1, sigma1: 真实图像特征的均值和协方差
    mu2, sigma2: 生成图像特征的均值和协方差
    """
    diff = mu1 - mu2
    covmean, _ = linalg.sqrtm(sigma1 @ sigma2, disp=False)

    # 处理数值不稳定产生的复数部分
    if np.iscomplexobj(covmean):
        covmean = covmean.real

    fid = diff @ diff + np.trace(sigma1) + np.trace(sigma2) - 2 * np.trace(covmean)
    return float(fid)

# 示例: mu shape=(2048,), sigma shape=(2048, 2048)
fid_value = calculate_fid(mu_real, sigma_real, mu_fake, sigma_fake)
print(f"FID: {fid_value:.2f}")  # 越低越好

import torch
import numpy as np
from torch.utils.data import DataLoader

@torch.no_grad()
def compute_fid_features(
    model: torch.nn.Module,
    dataloader: DataLoader,
    device: str = "cuda"
) -> tuple:
    """从数据加载器提取特征并计算统计量"""
    features_list = []
    model.eval()

    for batch in dataloader:
        images = batch[0].to(device)
        if images.shape[1] == 1:
            images = images.repeat(1, 3, 1, 1)  # 灰度转 RGB
        images = F.interpolate(images, size=(299, 299), mode="bilinear")
        features = model(images)
        features_list.append(features.cpu().numpy())

    all_features = np.concatenate(features_list, axis=0)
    mu = np.mean(all_features, axis=0)
    sigma = np.cov(all_features, rowvar=False)
    return mu, sigma

# 完整 FID 计算
mu_r, sigma_r = compute_fid_features(inception, real_loader)
mu_f, sigma_f = compute_fid_features(inception, fake_loader)
fid = calculate_fid(mu_r, sigma_r, mu_f, sigma_f)
print(f"Fréchet Inception Distance: {fid:.2f}")

import torch
import clip

# 加载 CLIP 模型
device = "cuda" if torch.cuda.is_available() else "cpu"
model, preprocess = clip.load("ViT-B/32", device=device)

@torch.no_grad()
def compute_clip_score(
    images: torch.Tensor,
    texts: list[str],
    batch_size: int = 32
) -> torch.Tensor:
    """
    计算批量图像-文本对的 CLIP Score
    images: (N, 3, 224, 224) 预处理后的图像
    texts: N 个文本提示
    """
    text_tokens = clip.tokenize(texts).to(device)
    image_features = model.encode_image(images)
    text_features = model.encode_text(text_tokens)

    # 归一化后计算余弦相似度
    image_features = image_features / image_features.norm(dim=-1, keepdim=True)
    text_features = text_features / text_features.norm(dim=-1, keepdim=True)

    scores = (image_features * text_features).sum(dim=-1)
    # 缩放到 [0, 100] 范围
    scores = scores * 100.0
    return scores

scores = compute_clip_score(image_batch, text_prompts)
print(f"Mean CLIP Score: {scores.mean().item():.2f}")

from transformers import CLIPProcessor, CLIPModel
import torch

# 使用 HuggingFace transformers 的替代方案
processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
clip_model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")

def evaluate_t2i_model(
    generator, prompts: list[str],
    images_per_prompt: int = 5
) -> dict:
    """全面评估文本到图像生成模型"""
    all_scores = []

    for prompt in prompts:
        images = generator.generate(prompt, num_images=images_per_prompt)
        inputs = processor(
            text=[prompt] * len(images),
            images=images,
            return_tensors="pt",
            padding=True
        )
        outputs = clip_model(**inputs)
        logits_per_image = outputs.logits_per_image
        scores = torch.diag(logits_per_image).numpy()
        all_scores.extend(scores.tolist())

    return {
        "mean_clip_score": np.mean(all_scores),
        "std_clip_score": np.std(all_scores),
        "min_score": np.min(all_scores),
        "max_score": np.max(all_scores)
    }

results = evaluate_t2i_model(my_generator, test_prompts)
print(f"CLIP Score: {results['mean_clip_score']:.2f} +/- {results['std_clip_score']:.2f}")

import torch

def compute_precision_recall(
    real_features: torch.Tensor,
    fake_features: torch.Tensor,
    k: int = 5
) -> tuple:
    """
    计算生成模型的 Precision 和 Recall
    基于 K 近邻的流形估计方法
    """
    # 计算真实特征到自身第 k 近邻的距离
    real_dists = torch.cdist(real_features, real_features)
    real_dists = torch.sort(real_dists, dim=1)[0]
    real_radii = real_dists[:, k]  # 第 k 近邻距离

    # 计算生成特征到真实特征的距离
    fake_to_real = torch.cdist(fake_features, real_features)
    # Precision: 生成样本有多少在真实流形内
    precision = (fake_to_real < real_radii.unsqueeze(0)).any(dim=1).float().mean()

    # 计算生成特征到自身第 k 近邻的距离
    fake_dists = torch.cdist(fake_features, fake_features)
    fake_dists = torch.sort(fake_dists, dim=1)[0]
    fake_radii = fake_dists[:, k]
    # Recall: 真实样本有多少被生成分布覆盖
    real_to_fake = torch.cdist(real_features, fake_features)
    recall = (real_to_fake < fake_radii.unsqueeze(0)).any(dim=1).float().mean()

    return precision.item(), recall.item()

p, r = compute_precision_recall(real_feats, fake_feats, k=5)
print(f"Precision: {p:.4f}, Recall: {r:.4f}")

import numpy as np
import torch

class ImprovedPrecisionRecall:
    """改进的 Precision/Recall 计算(Mani 等人 2020)"""

    def __init__(self, k: int = 5, num_center: int = 20000):
        self.k = k
        self.num_center = num_center

    def _compute_nearest_distances(self, X, Y):
        """计算 X 中每个点到 Y 的第 k 近邻距离"""
        dists = torch.cdist(X, Y)
        knn_dists = torch.topk(dists, k=self.k + 1, largest=False)[0]
        return knn_dists[:, -1]  # 第 k 近邻距离

    def fit(self, real_features: torch.Tensor, fake_features: torch.Tensor):
        """拟合真实和生成分布的流形"""
        r_idx = torch.randperm(len(real_features))[:self.num_center]
        f_idx = torch.randperm(len(fake_features))[:self.num_center]

        self.real_centers = real_features[r_idx]
        self.fake_centers = fake_features[f_idx]

        self.real_radii = self._compute_nearest_distances(
            self.real_centers, real_features
        )
        self.fake_radii = self._compute_nearest_distances(
            self.fake_centers, fake_features
        )

    def compute(self, real_features, fake_features):
        """计算 Precision 和 Recall"""
        precision = self._compute_fraction_inside(
            fake_features, self.real_centers, self.real_radii
        )
        recall = self._compute_fraction_inside(
            real_features, self.fake_centers, self.fake_radii
        )
        return precision, recall

pr = ImprovedPrecisionRecall(k=5)
pr.fit(real_feats, fake_feats)
precision, recall = pr.compute(real_feats, fake_feats)
f_score = 2 * precision * recall / (precision + recall + 1e-8)
print(f"P={precision:.3f}, R={recall:.3f}, F={f_score:.3f}")

import csv
from dataclasses import dataclass
from typing import List

@dataclass
class HumanEvalSample:
    image_id: str
    prompt: str
    mos_score: float      # 1-5 平均意见评分
    realism_score: float  # 1-5 真实度评分
    alignment_score: float  # 1-5 图文一致性评分
    is_real_guess: bool   # 人类猜是否真实

def analyze_human_eval(samples: List[HumanEvalSample]) -> dict:
    """分析人工评估结果"""
    n = len(samples)
    mos = sum(s.mos_score for s in samples) / n
    realism = sum(s.realism_score for s in samples) / n
    alignment = sum(s.alignment_score for s in samples) / n
    # 人类辨别准确率（理想情况应接近 50%）
    accuracy = sum(1 for s in samples if s.is_real_guess) / n

    return {
        "num_evaluations": n,
        "mean_opinion_score": round(mos, 3),
        "mean_realism_score": round(realism, 3),
        "mean_alignment_score": round(alignment, 3),
        "human_accuracy": round(accuracy, 3)
    }

# 从 CSV 读取评估结果
results = []
with open("human_eval_results.csv") as f:
    reader = csv.DictReader(f)
    for row in reader:
        results.append(HumanEvalSample(**row))

stats = analyze_human_eval(results)
print(f"MOS: {stats['mean_opinion_score']}, Human Acc: {stats['human_accuracy']}")

import numpy as np
from scipy import stats

def pairwise_preference_analysis(
    preferences: np.ndarray,
    model_names: list[str]
) -> dict:
    """
    分析两两比较实验结果
    preferences[i,j] = model_i 胜过 model_j 的次数
    """
    n_models = len(model_names)
    total_comparisons = preferences + preferences.T

    # Elo 风格的胜率计算
    win_rates = np.zeros(n_models)
    for i in range(n_models):
        for j in range(n_models):
            if i != j and total_comparisons[i, j] > 0:
                win_rates[i] += preferences[i, j] / total_comparisons[i, j]
    win_rates /= (n_models - 1)

    # 统计显著性检验
    significance = {}
    for i in range(n_models):
        for j in range(i + 1, n_models):
            n = total_comparisons[i, j]
            if n > 0:
                p_value = stats.binom_test(
                    preferences[i, j], n, p=0.5
                )
                significance[f"{model_names[i]} vs {model_names[j]}"] = {
                    "preference_ratio": preferences[i, j] / n,
                    "p_value": round(p_value, 4),
                    "significant": p_value < 0.05
                }

    return {"win_rates": dict(zip(model_names, win_rates)), "significance": significance}

# 示例: preferences[i,j] 表示模型 i 胜过模型 j 的次数
prefs = np.array([[0, 60, 45], [40, 0, 30], [55, 70, 0]])
result = pairwise_preference_analysis(prefs, ["ModelA", "ModelB", "ModelC"])
print(result)

import torch
from torchmetrics.image.fid import FrechetInceptionDistance
from torchmetrics.image.kid import KernelInceptionDistance
from torchmetrics.image.inception import InceptionScore

# FID 计算
fid = FrechetInceptionDistance(feature=2048)
fid.update(real_images, real=True)
fid.update(fake_images, real=False)
fid_score = fid.compute()
print(f"FID: {fid_score:.2f}")

# KID (Kernel Inception Distance) - 无偏估计
kid = KernelInceptionDistance(subset_size=1000)
kid.update(real_images, real=True)
kid.update(fake_images, real=False)
kid_mean, kid_std = kid.compute()
print(f"KID: {kid_mean:.4f} +/- {kid_std:.4f}")

# Inception Score
inception = InceptionScore()
inception.update(fake_images)
is_mean, is_std = inception.compute()
print(f"IS: {is_mean:.2f} +/- {is_std:.2f}")

import torch
from torchmetrics.image.fid import FrechetInceptionDistance
from torch.utils.tensorboard import SummaryWriter
import os

class GenerationEvaluator:
    """集成化的生成模型评估器"""

    def __init__(self, device: str = "cuda", log_dir: str = "logs"):
        self.device = device
        self.fid = FrechetInceptionDistance(feature=2048).to(device)
        self.writer = SummaryWriter(log_dir)
        os.makedirs("checkpoints", exist_ok=True)

    def evaluate_epoch(
        self,
        generator: torch.nn.Module,
        real_loader: torch.utils.data.DataLoader,
        epoch: int,
        num_samples: int = 5000
    ) -> dict:
        """每个 epoch 评估生成模型"""
        generator.eval()

        # 更新真实数据特征
        for batch in real_loader:
            self.fid.update(batch[0].to(self.device), real=True)

        # 生成图像并更新
        with torch.no_grad():
            noise = torch.randn(num_samples, 100, device=self.device)
            fake_images = generator(noise)
            self.fid.update(fake_images, real=False)

        fid_score = self.fid.compute().item()

        # 记录到 TensorBoard
        self.writer.add_scalar("Metrics/FID", fid_score, epoch)
        self.writer.add_images("Generated", fake_images[:8], epoch)

        # 保存最佳模型
        if fid_score < getattr(self, "best_fid", float("inf")):
            self.best_fid = fid_score
            torch.save(generator.state_dict(), "checkpoints/best_generator.pt")
            print(f"New best FID: {fid_score:.2f} at epoch {epoch}")

        self.fid.reset()
        return {"fid": fid_score, "best_fid": self.best_fid}

evaluator = GenerationEvaluator(device="cuda")
for epoch in range(num_epochs):
    train_one_epoch(generator, dataloader, epoch)
    metrics = evaluator.evaluate_epoch(generator, real_loader, epoch)
    print(f"Epoch {epoch}: FID={metrics['fid']:.2f}")