机器人互联网：当 57.5% 的流量来自 AI 爬虫，网站如何适应？

# 允许善意爬虫
User-agent: Googlebot
User-agent: Bingbot
Allow: /

# 限制 AI 训练爬虫
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
Crawl-delay: 10

# 屏蔽已知恶意爬虫
User-agent: *
Disallow: /api/
Disallow: /admin/

# 指向 AI 爬虫政策
AI-Training-Policy: https://example.com/ai-policy

// Cloudflare Worker: 检测并分类机器人流量
export default {
  async fetch(request, env) {
    const userAgent = request.headers.get('User-Agent') || '';
    const ip = request.headers.get('CF-Connecting-IP');
    
    // 分类机器人流量
    const botCategories = {
      training: ['GPTBot', 'ClaudeBot', 'Google-Extended'],
      aiAgent: ['OpenClaw', 'Perplexity', 'BingChat'],
      search: ['Googlebot', 'Bingbot', 'YandexBot'],
      malicious: ['python-requests', 'curl', 'wget']
    };
    
    let category = 'human';
    for (const [cat, patterns] of Object.entries(botCategories)) {
      if (patterns.some(p => userAgent.includes(p))) {
        category = cat;
        break;
      }
    }
    
    // 根据分类执行不同策略
    if (category === 'malicious') {
      return new Response('Access Denied', { status: 403 });
    }
    
    if (category === 'training' && env.PAY_PER_CRAWL) {
      // 向训练爬虫收费
      const cost = await calculateCrawlCost(request);
      return new Response(JSON.stringify({ cost, terms: '...' }), {
        headers: { 'Content-Type': 'application/json' }
      });
    }
    
    // 正常响应
    return fetch(request);
  }
};